hvac-kia-content/test_social_media_competitive.py

#!/usr/bin/env python3
"""
Test script for Social Media Competitive Intelligence
Tests YouTube and Instagram competitive scrapers
"""

import os
import sys
import logging
from pathlib import Path

# Add src to Python path
sys.path.insert(0, str(Path(__file__).parent / "src"))

from competitive_intelligence.competitive_orchestrator import CompetitiveIntelligenceOrchestrator


def setup_logging():
    """Setup logging for testing."""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )


def test_orchestrator_initialization():
    """Test that the orchestrator initializes with social media scrapers."""
    print("🧪 Testing Competitive Intelligence Orchestrator Initialization")
    print("=" * 60)
    
    data_dir = Path("data")
    logs_dir = Path("logs")
    
    try:
        orchestrator = CompetitiveIntelligenceOrchestrator(data_dir, logs_dir)
        
        print(f"✅ Orchestrator initialized successfully")
        print(f"📊 Total scrapers: {len(orchestrator.scrapers)}")
        
        # Check for social media scrapers
        social_media_scrapers = [k for k in orchestrator.scrapers.keys() if k.startswith(('youtube_', 'instagram_'))]
        youtube_scrapers = [k for k in orchestrator.scrapers.keys() if k.startswith('youtube_')]
        instagram_scrapers = [k for k in orchestrator.scrapers.keys() if k.startswith('instagram_')]
        
        print(f"📱 Social media scrapers: {len(social_media_scrapers)}")
        print(f"🎥 YouTube scrapers: {len(youtube_scrapers)}")
        print(f"📸 Instagram scrapers: {len(instagram_scrapers)}")
        
        print("\nAvailable scrapers:")
        for scraper_name in sorted(orchestrator.scrapers.keys()):
            print(f"  • {scraper_name}")
        
        return orchestrator, True
        
    except Exception as e:
        print(f"❌ Failed to initialize orchestrator: {e}")
        return None, False


def test_list_competitors(orchestrator):
    """Test listing competitors."""
    print("\n🧪 Testing List Competitors")
    print("=" * 40)
    
    try:
        results = orchestrator.list_available_competitors()
        
        print(f"✅ Listed competitors successfully")
        print(f"📊 Total scrapers: {results['total_scrapers']}")
        
        for platform, competitors in results['by_platform'].items():
            if competitors:
                print(f"\n{platform.upper()}: {len(competitors)} scrapers")
                for competitor in competitors:
                    print(f"  • {competitor}")
        
        return True
        
    except Exception as e:
        print(f"❌ Failed to list competitors: {e}")
        return False


def test_social_media_status(orchestrator):
    """Test social media status."""
    print("\n🧪 Testing Social Media Status")
    print("=" * 40)
    
    try:
        results = orchestrator.get_social_media_status()
        
        print(f"✅ Got social media status successfully")
        print(f"📱 Total social media scrapers: {results['total_social_media_scrapers']}")
        print(f"🎥 YouTube scrapers: {results['youtube_scrapers']}")
        print(f"📸 Instagram scrapers: {results['instagram_scrapers']}")
        
        # Show status of each scraper
        for scraper_name, status in results['scrapers'].items():
            scraper_type = status.get('scraper_type', 'unknown')
            configured = status.get('scraper_configured', False)
            emoji = '✅' if configured else '❌'
            print(f"\n{emoji} {scraper_name} ({scraper_type}):")
            
            if 'error' in status:
                print(f"  ❌ Error: {status['error']}")
            else:
                # Show basic info
                if scraper_type == 'youtube':
                    metadata = status.get('channel_metadata', {})
                    print(f"  🏷️  Channel: {metadata.get('title', 'Unknown')}")
                    print(f"  👥 Subscribers: {metadata.get('subscriber_count', 'Unknown'):,}")
                elif scraper_type == 'instagram':
                    metadata = status.get('profile_metadata', {})
                    print(f"  🏷️  Account: {metadata.get('full_name', 'Unknown')}")
                    print(f"  👥 Followers: {metadata.get('followers', 'Unknown'):,}")
        
        return True
        
    except Exception as e:
        print(f"❌ Failed to get social media status: {e}")
        return False


def test_competitive_setup(orchestrator):
    """Test competitive setup."""
    print("\n🧪 Testing Competitive Setup")
    print("=" * 40)
    
    try:
        results = orchestrator.test_competitive_setup()
        
        overall_status = results.get('overall_status', 'unknown')
        print(f"Overall Status: {'✅' if overall_status == 'operational' else '❌'} {overall_status}")
        
        # Show test results for each scraper
        for scraper_name, test_result in results.get('test_results', {}).items():
            status = test_result.get('status', 'unknown')
            emoji = '✅' if status == 'success' else '❌'
            print(f"\n{emoji} {scraper_name}:")
            
            if status == 'success':
                config = test_result.get('config', {})
                print(f"  🌐 Base URL: {config.get('base_url', 'Unknown')}")
                print(f"  🔒 Proxy: {'✅' if config.get('proxy_configured') else '❌'}")
                print(f"  🤖 Jina AI: {'✅' if config.get('jina_api_configured') else '❌'}")
                print(f"  📁 Directories: {'✅' if config.get('directories_exist') else '❌'}")
            else:
                print(f"  ❌ Error: {test_result.get('error', 'Unknown')}")
        
        return overall_status == 'operational'
        
    except Exception as e:
        print(f"❌ Failed to test competitive setup: {e}")
        return False


def test_youtube_discovery(orchestrator):
    """Test YouTube content discovery (dry run)."""
    print("\n🧪 Testing YouTube Content Discovery")
    print("=" * 40)
    
    youtube_scrapers = {k: v for k, v in orchestrator.scrapers.items() if k.startswith('youtube_')}
    
    if not youtube_scrapers:
        print("⚠️  No YouTube scrapers available")
        return False
    
    # Test one YouTube scraper
    scraper_name = list(youtube_scrapers.keys())[0]
    scraper = youtube_scrapers[scraper_name]
    
    try:
        print(f"🎥 Testing content discovery for {scraper_name}")
        
        # Discover a small number of URLs
        content_urls = scraper.discover_content_urls(3)
        
        print(f"✅ Discovered {len(content_urls)} content URLs")
        
        for i, url_data in enumerate(content_urls, 1):
            url = url_data.get('url') if isinstance(url_data, dict) else url_data
            title = url_data.get('title', 'Unknown') if isinstance(url_data, dict) else 'Unknown'
            print(f"  {i}. {title[:50]}...")
            print(f"     {url}")
        
        return True
        
    except Exception as e:
        print(f"❌ YouTube discovery test failed: {e}")
        return False


def test_instagram_discovery(orchestrator):
    """Test Instagram content discovery (dry run)."""
    print("\n🧪 Testing Instagram Content Discovery")
    print("=" * 40)
    
    instagram_scrapers = {k: v for k, v in orchestrator.scrapers.items() if k.startswith('instagram_')}
    
    if not instagram_scrapers:
        print("⚠️  No Instagram scrapers available")
        return False
    
    # Test one Instagram scraper
    scraper_name = list(instagram_scrapers.keys())[0]
    scraper = instagram_scrapers[scraper_name]
    
    try:
        print(f"📸 Testing content discovery for {scraper_name}")
        
        # Discover a small number of URLs
        content_urls = scraper.discover_content_urls(2)  # Very small for Instagram
        
        print(f"✅ Discovered {len(content_urls)} content URLs")
        
        for i, url_data in enumerate(content_urls, 1):
            url = url_data.get('url') if isinstance(url_data, dict) else url_data
            caption = url_data.get('caption', '')[:30] + '...' if isinstance(url_data, dict) and url_data.get('caption') else 'No caption'
            print(f"  {i}. {caption}")
            print(f"     {url}")
        
        return True
        
    except Exception as e:
        print(f"❌ Instagram discovery test failed: {e}")
        return False


def main():
    """Run all tests."""
    setup_logging()
    
    print("🧪 Social Media Competitive Intelligence Test Suite")
    print("=" * 60)
    print("This test suite validates the Phase 2 social media competitive scrapers")
    print()
    
    # Test 1: Orchestrator initialization
    orchestrator, init_success = test_orchestrator_initialization()
    if not init_success:
        print("❌ Critical failure: Could not initialize orchestrator")
        sys.exit(1)
    
    test_results = {'initialization': True}
    
    # Test 2: List competitors
    test_results['list_competitors'] = test_list_competitors(orchestrator)
    
    # Test 3: Social media status
    test_results['social_media_status'] = test_social_media_status(orchestrator)
    
    # Test 4: Competitive setup
    test_results['competitive_setup'] = test_competitive_setup(orchestrator)
    
    # Test 5: YouTube discovery (only if API key available)
    if os.getenv('YOUTUBE_API_KEY'):
        test_results['youtube_discovery'] = test_youtube_discovery(orchestrator)
    else:
        print("\n⚠️  Skipping YouTube discovery test (no API key)")
        test_results['youtube_discovery'] = None
    
    # Test 6: Instagram discovery (only if credentials available)
    if os.getenv('INSTAGRAM_USERNAME') and os.getenv('INSTAGRAM_PASSWORD'):
        test_results['instagram_discovery'] = test_instagram_discovery(orchestrator)
    else:
        print("\n⚠️  Skipping Instagram discovery test (no credentials)")
        test_results['instagram_discovery'] = None
    
    # Summary
    print("\n" + "=" * 60)
    print("📋 TEST SUMMARY")
    print("=" * 60)
    
    passed = sum(1 for result in test_results.values() if result is True)
    failed = sum(1 for result in test_results.values() if result is False)
    skipped = sum(1 for result in test_results.values() if result is None)
    
    print(f"✅ Tests Passed: {passed}")
    print(f"❌ Tests Failed: {failed}")
    print(f"⚠️  Tests Skipped: {skipped}")
    
    for test_name, result in test_results.items():
        if result is True:
            print(f"  ✅ {test_name}")
        elif result is False:
            print(f"  ❌ {test_name}")
        else:
            print(f"  ⚠️  {test_name} (skipped)")
    
    if failed > 0:
        print(f"\n❌ Some tests failed. Check the logs above for details.")
        sys.exit(1)
    else:
        print(f"\n✅ All available tests passed! Social media competitive intelligence is ready.")
        print("\nNext steps:")
        print("1. Set up environment variables (YOUTUBE_API_KEY, INSTAGRAM_USERNAME, INSTAGRAM_PASSWORD)")
        print("2. Test backlog capture: python run_competitive_intelligence.py --operation social-backlog --limit 5")
        print("3. Test incremental sync: python run_competitive_intelligence.py --operation social-incremental")
        sys.exit(0)


if __name__ == "__main__":
    main()