0a795437a7
- Increased Instagram rate limit from 100 to 200 posts/hour - Reduced delays: 10-20s (was 15-30s), extended breaks 30-60s (was 60-120s) - Extended break interval: every 10 requests (was 5) - Updated capture targets: 1000 posts for Instagram, 1000 videos for TikTok - Added production deployment and monitoring scripts - Created environment configuration template This provides ~40-50% speed improvement for Instagram scraping and captures 5x more Instagram content and 3.3x more TikTok content. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
197 lines
No EOL
6.5 KiB
Bash
Executable file
197 lines
No EOL
6.5 KiB
Bash
Executable file
#!/bin/bash
|
|
#
|
|
# Production Validation Script
|
|
# Tests all production components and services
|
|
#
|
|
|
|
set -e
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
PROD_DIR="/opt/hvac-kia-content"
|
|
SERVICE_USER="hvac-content"
|
|
|
|
# Print colored output
|
|
print_status() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
print_success() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|
print_warning() { echo -e "${YELLOW}[!]${NC} $1"; }
|
|
print_error() { echo -e "${RED}[✗]${NC} $1"; }
|
|
|
|
# Test counters
|
|
TESTS_PASSED=0
|
|
TESTS_FAILED=0
|
|
|
|
# Test function wrapper
|
|
run_test() {
|
|
local test_name="$1"
|
|
local test_command="$2"
|
|
|
|
echo -n "Testing $test_name... "
|
|
|
|
if eval "$test_command" >/dev/null 2>&1; then
|
|
print_success "$test_name"
|
|
((TESTS_PASSED++))
|
|
return 0
|
|
else
|
|
print_error "$test_name"
|
|
((TESTS_FAILED++))
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Test production directory structure
|
|
test_directories() {
|
|
print_status "Validating directory structure..."
|
|
|
|
run_test "Production directory exists" "[[ -d '$PROD_DIR' ]]"
|
|
run_test "Data directory exists" "[[ -d '$PROD_DIR/data' ]]"
|
|
run_test "Logs directory exists" "[[ -d '$PROD_DIR/logs' ]]"
|
|
run_test "Source code exists" "[[ -d '$PROD_DIR/src' ]]"
|
|
run_test "Config directory exists" "[[ -d '$PROD_DIR/config' ]]"
|
|
run_test "Virtual environment exists" "[[ -d '$PROD_DIR/venv' ]]"
|
|
}
|
|
|
|
# Test service user
|
|
test_service_user() {
|
|
print_status "Validating service user..."
|
|
|
|
run_test "Service user exists" "id '$SERVICE_USER'"
|
|
run_test "Service user home directory" "[[ -d '/home/$SERVICE_USER' || '$PROD_DIR' ]]"
|
|
run_test "Production directory ownership" "[[ \$(stat -c '%U' '$PROD_DIR') == '$SERVICE_USER' ]]"
|
|
}
|
|
|
|
# Test Python environment
|
|
test_python_environment() {
|
|
print_status "Validating Python environment..."
|
|
|
|
run_test "Python virtual environment" "[[ -f '$PROD_DIR/venv/bin/python' ]]"
|
|
run_test "Python packages installed" "sudo -u '$SERVICE_USER' '$PROD_DIR/venv/bin/pip' list | grep -q 'requests'"
|
|
run_test "Source modules importable" "sudo -u '$SERVICE_USER' '$PROD_DIR/venv/bin/python' -c 'import src.orchestrator'"
|
|
run_test "Playwright browser installed" "sudo -u '$SERVICE_USER' '$PROD_DIR/venv/bin/python' -c 'from playwright.sync_api import sync_playwright; sync_playwright().chromium.executable_path'"
|
|
}
|
|
|
|
# Test systemd services
|
|
test_systemd_services() {
|
|
print_status "Validating systemd services..."
|
|
|
|
run_test "Main service file exists" "[[ -f '/etc/systemd/system/hvac-content-aggregator.service' ]]"
|
|
run_test "Main timer file exists" "[[ -f '/etc/systemd/system/hvac-content-aggregator.timer' ]]"
|
|
run_test "Monitoring service exists" "[[ -f '/etc/systemd/system/hvac-monitoring.service' ]]"
|
|
run_test "TikTok service exists" "[[ -f '/etc/systemd/system/hvac-tiktok-captions.service' ]]"
|
|
run_test "Main timer enabled" "systemctl is-enabled hvac-content-aggregator.timer"
|
|
run_test "Monitoring timer enabled" "systemctl is-enabled hvac-monitoring.timer"
|
|
}
|
|
|
|
# Test configuration files
|
|
test_configuration() {
|
|
print_status "Validating configuration..."
|
|
|
|
run_test "Production config exists" "[[ -f '$PROD_DIR/config/production.py' ]]"
|
|
run_test "Environment template exists" "[[ -f '$PROD_DIR/.env.template' ]]"
|
|
run_test "Main runner script exists" "[[ -f '$PROD_DIR/run_production.py' ]]"
|
|
run_test "Backlog capture script exists" "[[ -f '$PROD_DIR/production_backlog_capture.py' ]]"
|
|
}
|
|
|
|
# Test network connectivity
|
|
test_connectivity() {
|
|
print_status "Validating network connectivity..."
|
|
|
|
run_test "HVAC Know It All website reachable" "curl -s --max-time 10 https://hvacknowitall.com > /dev/null"
|
|
run_test "YouTube accessible" "curl -s --max-time 10 https://www.youtube.com/@HVACKnowItAll > /dev/null"
|
|
run_test "MailChimp RSS accessible" "curl -s --max-time 10 'https://us10.campaign-archive.com/feed?u=d1a98c3e62003104038942e21&id=2205dbf985' > /dev/null"
|
|
}
|
|
|
|
# Test dry run execution
|
|
test_dry_run() {
|
|
print_status "Validating application execution..."
|
|
|
|
if [[ -f "$PROD_DIR/.env" ]]; then
|
|
run_test "Production script dry run" "sudo -u '$SERVICE_USER' bash -c 'cd $PROD_DIR && source venv/bin/activate && timeout 30 python run_production.py --dry-run'"
|
|
else
|
|
print_warning "Skipping dry run test - .env file not configured"
|
|
fi
|
|
}
|
|
|
|
# Test log rotation
|
|
test_log_rotation() {
|
|
print_status "Validating log rotation..."
|
|
|
|
run_test "Logrotate configuration exists" "[[ -f '/etc/logrotate.d/hvac-content' ]]"
|
|
run_test "Logrotate configuration syntax" "logrotate -d /etc/logrotate.d/hvac-content"
|
|
}
|
|
|
|
# Test monitoring
|
|
test_monitoring() {
|
|
print_status "Validating monitoring setup..."
|
|
|
|
run_test "Monitoring directory exists" "[[ -d '$PROD_DIR/monitoring' ]]"
|
|
run_test "Dashboard generator exists" "[[ -f '$PROD_DIR/monitoring/dashboard_generator.py' ]]"
|
|
}
|
|
|
|
# Main validation function
|
|
main() {
|
|
echo "🔍 HVAC Know It All - Production Validation"
|
|
echo "=========================================="
|
|
echo
|
|
|
|
test_directories
|
|
echo
|
|
test_service_user
|
|
echo
|
|
test_python_environment
|
|
echo
|
|
test_systemd_services
|
|
echo
|
|
test_configuration
|
|
echo
|
|
test_connectivity
|
|
echo
|
|
test_dry_run
|
|
echo
|
|
test_log_rotation
|
|
echo
|
|
test_monitoring
|
|
|
|
echo
|
|
echo "=========================================="
|
|
if [[ $TESTS_FAILED -eq 0 ]]; then
|
|
print_success "🎉 All tests passed! ($TESTS_PASSED/$((TESTS_PASSED + TESTS_FAILED)))"
|
|
print_status "Production environment is ready for deployment"
|
|
exit 0
|
|
else
|
|
print_error "⚠️ Some tests failed: $TESTS_FAILED/$((TESTS_PASSED + TESTS_FAILED))"
|
|
print_status "Please address the failed tests before proceeding"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Show help
|
|
if [[ "$1" == "--help" || "$1" == "-h" ]]; then
|
|
echo "HVAC Know It All - Production Validation Script"
|
|
echo
|
|
echo "Usage: $0 [options]"
|
|
echo
|
|
echo "This script validates the production deployment by testing:"
|
|
echo " • Directory structure and permissions"
|
|
echo " • Service user configuration"
|
|
echo " • Python environment and dependencies"
|
|
echo " • Systemd services and timers"
|
|
echo " • Configuration files"
|
|
echo " • Network connectivity"
|
|
echo " • Application execution (dry run)"
|
|
echo " • Log rotation setup"
|
|
echo " • Monitoring configuration"
|
|
echo
|
|
echo "Options:"
|
|
echo " -h, --help Show this help message"
|
|
echo
|
|
exit 0
|
|
fi
|
|
|
|
# Run validation
|
|
main "$@" |