import os
import logging
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any, Union

import pytz

from .hvacrschool_competitive_scraper import HVACRSchoolCompetitiveScraper
from .youtube_competitive_scraper import create_youtube_competitive_scrapers
from .instagram_competitive_scraper import create_instagram_competitive_scrapers
from .exceptions import (
    CompetitiveIntelligenceError, ConfigurationError, QuotaExceededError,
    YouTubeAPIError, InstagramError, RateLimitError
)
from .types import Platform, OperationResult


class CompetitiveIntelligenceOrchestrator:
    """Orchestrator for competitive intelligence scraping operations."""
    
    def __init__(self, data_dir: Path, logs_dir: Path):
        """Initialize the competitive intelligence orchestrator."""
        self.data_dir = data_dir
        self.logs_dir = logs_dir
        self.tz = pytz.timezone(os.getenv('TIMEZONE', 'America/Halifax'))
        
        # Setup logging
        self.logger = self._setup_logger()
        
        # Initialize competitive scrapers
        self.scrapers = {
            'hvacrschool': HVACRSchoolCompetitiveScraper(data_dir, logs_dir)
        }
        
        # Add YouTube competitive scrapers
        try:
            youtube_scrapers = create_youtube_competitive_scrapers(data_dir, logs_dir)
            self.scrapers.update(youtube_scrapers)
            self.logger.info(f"Initialized {len(youtube_scrapers)} YouTube competitive scrapers")
        except (ConfigurationError, YouTubeAPIError) as e:
            self.logger.error(f"Configuration error initializing YouTube scrapers: {e}")
        except Exception as e:
            self.logger.error(f"Unexpected error initializing YouTube scrapers: {e}")
        
        # Add Instagram competitive scrapers
        try:
            instagram_scrapers = create_instagram_competitive_scrapers(data_dir, logs_dir)
            self.scrapers.update(instagram_scrapers)
            self.logger.info(f"Initialized {len(instagram_scrapers)} Instagram competitive scrapers")
        except (ConfigurationError, InstagramError) as e:
            self.logger.error(f"Configuration error initializing Instagram scrapers: {e}")
        except Exception as e:
            self.logger.error(f"Unexpected error initializing Instagram scrapers: {e}")
        
        # Execution tracking
        self.execution_results = {}
        
        self.logger.info(f"Competitive Intelligence Orchestrator initialized with {len(self.scrapers)} scrapers")
        self.logger.info(f"Available scrapers: {list(self.scrapers.keys())}")

    def _setup_logger(self) -> logging.Logger:
        """Setup orchestrator logger."""
        logger = logging.getLogger("competitive_intelligence_orchestrator")
        logger.setLevel(logging.INFO)
        
        # Console handler
        if not logger.handlers:  # Avoid duplicate handlers
            console_handler = logging.StreamHandler()
            console_handler.setLevel(logging.INFO)
            
            # File handler
            log_dir = self.logs_dir / "competitive_intelligence"
            log_dir.mkdir(parents=True, exist_ok=True)
            
            from logging.handlers import RotatingFileHandler
            file_handler = RotatingFileHandler(
                log_dir / "competitive_orchestrator.log",
                maxBytes=10 * 1024 * 1024,
                backupCount=5
            )
            file_handler.setLevel(logging.DEBUG)
            
            # Formatter
            formatter = logging.Formatter(
                '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                datefmt='%Y-%m-%d %H:%M:%S'
            )
            console_handler.setFormatter(formatter)
            file_handler.setFormatter(formatter)
            
            logger.addHandler(console_handler)
            logger.addHandler(file_handler)
        
        return logger

    def run_backlog_capture(self, 
                           competitors: Optional[List[str]] = None, 
                           limit_per_competitor: Optional[int] = None) -> Dict[str, any]:
        """Run backlog capture for specified competitors."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting competitive intelligence backlog capture at {start_time}")
        
        # Default to all competitors if none specified
        if competitors is None:
            competitors = list(self.scrapers.keys())
        
        # Validate competitors
        valid_competitors = [c for c in competitors if c in self.scrapers]
        if not valid_competitors:
            self.logger.error(f"No valid competitors found. Available: {list(self.scrapers.keys())}")
            return {'error': 'No valid competitors'}
        
        self.logger.info(f"Running backlog capture for competitors: {valid_competitors}")
        
        results = {}
        
        # Run backlog capture for each competitor sequentially (to be polite)
        for competitor in valid_competitors:
            try:
                self.logger.info(f"Starting backlog capture for {competitor}")
                scraper = self.scrapers[competitor]
                
                # Run backlog capture
                scraper.run_backlog_capture(limit_per_competitor)
                
                results[competitor] = {
                    'status': 'success',
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'message': f'Backlog capture completed for {competitor}'
                }
                
                self.logger.info(f"Completed backlog capture for {competitor}")
                
                # Brief pause between competitors
                time.sleep(5)
                
            except (QuotaExceededError, RateLimitError) as e:
                error_msg = f"Rate/quota limit error in backlog capture for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'rate_limited',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'retry_recommended': True
                }
            except (YouTubeAPIError, InstagramError) as e:
                error_msg = f"Platform-specific error in backlog capture for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'platform_error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
            except Exception as e:
                error_msg = f"Unexpected error in backlog capture for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        self.logger.info(f"Competitive backlog capture completed in {duration}")
        
        return {
            'operation': 'backlog_capture',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'competitors': valid_competitors,
            'results': results
        }

    def run_incremental_sync(self, 
                            competitors: Optional[List[str]] = None) -> Dict[str, any]:
        """Run incremental sync for specified competitors."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting competitive intelligence incremental sync at {start_time}")
        
        # Default to all competitors if none specified
        if competitors is None:
            competitors = list(self.scrapers.keys())
        
        # Validate competitors
        valid_competitors = [c for c in competitors if c in self.scrapers]
        if not valid_competitors:
            self.logger.error(f"No valid competitors found. Available: {list(self.scrapers.keys())}")
            return {'error': 'No valid competitors'}
        
        self.logger.info(f"Running incremental sync for competitors: {valid_competitors}")
        
        results = {}
        
        # Run incremental sync for each competitor
        for competitor in valid_competitors:
            try:
                self.logger.info(f"Starting incremental sync for {competitor}")
                scraper = self.scrapers[competitor]
                
                # Run incremental sync
                scraper.run_incremental_sync()
                
                results[competitor] = {
                    'status': 'success',
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'message': f'Incremental sync completed for {competitor}'
                }
                
                self.logger.info(f"Completed incremental sync for {competitor}")
                
                # Brief pause between competitors
                time.sleep(2)
                
            except (QuotaExceededError, RateLimitError) as e:
                error_msg = f"Rate/quota limit error in incremental sync for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'rate_limited',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'retry_recommended': True
                }
            except (YouTubeAPIError, InstagramError) as e:
                error_msg = f"Platform-specific error in incremental sync for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'platform_error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
            except Exception as e:
                error_msg = f"Unexpected error in incremental sync for {competitor}: {e}"
                self.logger.error(error_msg)
                results[competitor] = {
                    'status': 'error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        self.logger.info(f"Competitive incremental sync completed in {duration}")
        
        return {
            'operation': 'incremental_sync',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'competitors': valid_competitors,
            'results': results
        }

    def get_competitor_status(self, competitor: str = None) -> Dict[str, any]:
        """Get status information for competitors."""
        if competitor and competitor not in self.scrapers:
            return {'error': f'Unknown competitor: {competitor}'}
        
        status = {}
        
        # Get status for specific competitor or all
        competitors = [competitor] if competitor else list(self.scrapers.keys())
        
        for comp_name in competitors:
            try:
                scraper = self.scrapers[comp_name]
                comp_status = scraper.load_competitive_state()
                
                # Add runtime information
                comp_status['scraper_configured'] = True
                comp_status['base_url'] = scraper.base_url
                comp_status['proxy_enabled'] = bool(scraper.competitive_config.use_proxy and 
                                                   scraper.oxylabs_config.get('username'))
                
                status[comp_name] = comp_status
                
            except CompetitiveIntelligenceError as e:
                status[comp_name] = {
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'scraper_configured': False
                }
            except Exception as e:
                status[comp_name] = {
                    'error': str(e),
                    'error_type': 'UnexpectedError',
                    'scraper_configured': False
                }
        
        return status

    def run_competitive_analysis(self, competitors: Optional[List[str]] = None) -> Dict[str, any]:
        """Run competitive analysis workflow combining content capture and analysis."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting comprehensive competitive analysis at {start_time}")
        
        # Step 1: Run incremental sync
        sync_results = self.run_incremental_sync(competitors)
        
        # Step 2: Generate analysis report (placeholder for now)
        analysis_results = self._generate_competitive_analysis_report(competitors)
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        return {
            'operation': 'competitive_analysis',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'sync_results': sync_results,
            'analysis_results': analysis_results
        }

    def _generate_competitive_analysis_report(self, 
                                            competitors: Optional[List[str]] = None) -> Dict[str, any]:
        """Generate competitive analysis report (placeholder for Phase 3)."""
        self.logger.info("Generating competitive analysis report (Phase 3 feature)")
        
        # This is a placeholder for Phase 3 - Content Intelligence Analysis
        # Will integrate with Claude API for content analysis
        
        return {
            'status': 'planned_for_phase_3',
            'message': 'Content analysis will be implemented in Phase 3',
            'features_planned': [
                'Content topic analysis',
                'Publishing frequency analysis',
                'Content quality metrics',
                'Competitive positioning insights',
                'Content gap identification'
            ]
        }

    def cleanup_old_competitive_data(self, days_to_keep: int = 30) -> Dict[str, any]:
        """Clean up old competitive intelligence data."""
        self.logger.info(f"Cleaning up competitive data older than {days_to_keep} days")
        
        # This would implement cleanup logic for old competitive data
        # For now, just return a placeholder
        
        return {
            'status': 'not_implemented',
            'message': 'Cleanup functionality will be implemented as needed'
        }

    def test_competitive_setup(self) -> Dict[str, any]:
        """Test competitive intelligence setup."""
        self.logger.info("Testing competitive intelligence setup")
        
        test_results = {}
        
        # Test each scraper
        for competitor, scraper in self.scrapers.items():
            try:
                # Test basic configuration
                config_test = {
                    'base_url': scraper.base_url,
                    'proxy_configured': bool(scraper.oxylabs_config.get('username')),
                    'jina_api_configured': bool(scraper.jina_api_key),
                    'directories_exist': True
                }
                
                # Test directory structure
                comp_dir = self.data_dir / "competitive_intelligence" / competitor
                config_test['directories_exist'] = comp_dir.exists()
                
                # Test proxy connection (if configured)
                if config_test['proxy_configured']:
                    try:
                        response = scraper.session.get('http://httpbin.org/ip', timeout=10)
                        config_test['proxy_working'] = response.status_code == 200
                        if response.status_code == 200:
                            config_test['proxy_ip'] = response.json().get('origin', 'Unknown')
                    except Exception as e:
                        config_test['proxy_working'] = False
                        config_test['proxy_error'] = str(e)
                
                test_results[competitor] = {
                    'status': 'success',
                    'config': config_test
                }
                
            except Exception as e:
                test_results[competitor] = {
                    'status': 'error',
                    'error': str(e)
                }
        
        return {
            'overall_status': 'operational' if all(r.get('status') == 'success' for r in test_results.values()) else 'issues_detected',
            'test_results': test_results,
            'test_timestamp': datetime.now(self.tz).isoformat()
        }

    def run_social_media_backlog(self, 
                                platforms: Optional[List[str]] = None,
                                limit_per_competitor: Optional[int] = None) -> Dict[str, any]:
        """Run backlog capture specifically for social media competitors (YouTube, Instagram)."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting social media competitive backlog capture at {start_time}")
        
        # Filter for social media scrapers
        social_media_scrapers = {
            k: v for k, v in self.scrapers.items() 
            if k.startswith(('youtube_', 'instagram_'))
        }
        
        if platforms:
            # Further filter by platforms
            filtered_scrapers = {}
            for platform in platforms:
                platform_scrapers = {
                    k: v for k, v in social_media_scrapers.items()
                    if k.startswith(f'{platform}_')
                }
                filtered_scrapers.update(platform_scrapers)
            social_media_scrapers = filtered_scrapers
        
        if not social_media_scrapers:
            self.logger.error("No social media scrapers found")
            return {'error': 'No social media scrapers available'}
        
        self.logger.info(f"Running backlog for social media competitors: {list(social_media_scrapers.keys())}")
        
        results = {}
        
        # Run social media backlog capture sequentially (to be respectful)
        for scraper_name, scraper in social_media_scrapers.items():
            try:
                self.logger.info(f"Starting social media backlog for {scraper_name}")
                
                # Use smaller limits for social media
                limit = limit_per_competitor or (20 if scraper_name.startswith('instagram_') else 50)
                scraper.run_backlog_capture(limit)
                
                results[scraper_name] = {
                    'status': 'success',
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'message': f'Social media backlog completed for {scraper_name}',
                    'limit_used': limit
                }
                
                self.logger.info(f"Completed social media backlog for {scraper_name}")
                
                # Longer pause between social media scrapers
                time.sleep(10)
                
            except (QuotaExceededError, RateLimitError) as e:
                error_msg = f"Rate/quota limit in social media backlog for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'rate_limited',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'retry_recommended': True
                }
            except (YouTubeAPIError, InstagramError) as e:
                error_msg = f"Platform error in social media backlog for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'platform_error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
            except Exception as e:
                error_msg = f"Unexpected error in social media backlog for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        self.logger.info(f"Social media competitive backlog completed in {duration}")
        
        return {
            'operation': 'social_media_backlog',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'scrapers': list(social_media_scrapers.keys()),
            'results': results
        }

    def run_social_media_incremental(self, 
                                   platforms: Optional[List[str]] = None) -> Dict[str, any]:
        """Run incremental sync specifically for social media competitors."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting social media incremental sync at {start_time}")
        
        # Filter for social media scrapers
        social_media_scrapers = {
            k: v for k, v in self.scrapers.items() 
            if k.startswith(('youtube_', 'instagram_'))
        }
        
        if platforms:
            # Further filter by platforms
            filtered_scrapers = {}
            for platform in platforms:
                platform_scrapers = {
                    k: v for k, v in social_media_scrapers.items()
                    if k.startswith(f'{platform}_')
                }
                filtered_scrapers.update(platform_scrapers)
            social_media_scrapers = filtered_scrapers
        
        if not social_media_scrapers:
            self.logger.error("No social media scrapers found")
            return {'error': 'No social media scrapers available'}
        
        self.logger.info(f"Running incremental sync for social media: {list(social_media_scrapers.keys())}")
        
        results = {}
        
        # Run incremental sync for each social media scraper
        for scraper_name, scraper in social_media_scrapers.items():
            try:
                self.logger.info(f"Starting incremental sync for {scraper_name}")
                scraper.run_incremental_sync()
                
                results[scraper_name] = {
                    'status': 'success',
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'message': f'Social media incremental sync completed for {scraper_name}'
                }
                
                self.logger.info(f"Completed incremental sync for {scraper_name}")
                
                # Pause between social media scrapers
                time.sleep(5)
                
            except (QuotaExceededError, RateLimitError) as e:
                error_msg = f"Rate/quota limit in social incremental for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'rate_limited',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'retry_recommended': True
                }
            except (YouTubeAPIError, InstagramError) as e:
                error_msg = f"Platform error in social incremental for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'platform_error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
            except Exception as e:
                error_msg = f"Unexpected error in social incremental for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        self.logger.info(f"Social media incremental sync completed in {duration}")
        
        return {
            'operation': 'social_media_incremental',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'scrapers': list(social_media_scrapers.keys()),
            'results': results
        }

    def run_platform_analysis(self, platform: str) -> Dict[str, any]:
        """Run analysis for a specific platform (youtube or instagram)."""
        start_time = datetime.now(self.tz)
        self.logger.info(f"Starting {platform} competitive analysis at {start_time}")
        
        # Filter for platform scrapers
        platform_scrapers = {
            k: v for k, v in self.scrapers.items()
            if k.startswith(f'{platform}_')
        }
        
        if not platform_scrapers:
            return {'error': f'No {platform} scrapers found'}
        
        results = {}
        
        # Run analysis for each competitor on the platform
        for scraper_name, scraper in platform_scrapers.items():
            try:
                self.logger.info(f"Running analysis for {scraper_name}")
                
                # Check if scraper has competitor analysis method
                if hasattr(scraper, 'run_competitor_analysis'):
                    analysis = scraper.run_competitor_analysis()
                    results[scraper_name] = {
                        'status': 'success',
                        'analysis': analysis,
                        'timestamp': datetime.now(self.tz).isoformat()
                    }
                else:
                    results[scraper_name] = {
                        'status': 'not_supported',
                        'message': f'Analysis not supported for {scraper_name}'
                    }
                
                # Brief pause between analyses
                time.sleep(2)
                
            except (QuotaExceededError, RateLimitError) as e:
                error_msg = f"Rate/quota limit in analysis for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'rate_limited',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat(),
                    'retry_recommended': True
                }
            except (YouTubeAPIError, InstagramError) as e:
                error_msg = f"Platform error in analysis for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'platform_error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
            except Exception as e:
                error_msg = f"Unexpected error in analysis for {scraper_name}: {e}"
                self.logger.error(error_msg)
                results[scraper_name] = {
                    'status': 'error',
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'timestamp': datetime.now(self.tz).isoformat()
                }
        
        end_time = datetime.now(self.tz)
        duration = end_time - start_time
        
        return {
            'operation': f'{platform}_analysis',
            'start_time': start_time.isoformat(),
            'end_time': end_time.isoformat(),
            'duration_seconds': duration.total_seconds(),
            'platform': platform,
            'scrapers_analyzed': list(platform_scrapers.keys()),
            'results': results
        }

    def get_social_media_status(self) -> Dict[str, any]:
        """Get status specifically for social media competitive scrapers."""
        social_media_scrapers = {
            k: v for k, v in self.scrapers.items() 
            if k.startswith(('youtube_', 'instagram_'))
        }
        
        status = {
            'total_social_media_scrapers': len(social_media_scrapers),
            'youtube_scrapers': len([k for k in social_media_scrapers if k.startswith('youtube_')]),
            'instagram_scrapers': len([k for k in social_media_scrapers if k.startswith('instagram_')]),
            'scrapers': {}
        }
        
        for scraper_name, scraper in social_media_scrapers.items():
            try:
                # Get competitor metadata if available
                if hasattr(scraper, 'get_competitor_metadata'):
                    scraper_status = scraper.get_competitor_metadata()
                else:
                    scraper_status = scraper.load_competitive_state()
                
                scraper_status['scraper_type'] = 'youtube' if scraper_name.startswith('youtube_') else 'instagram'
                scraper_status['scraper_configured'] = True
                
                status['scrapers'][scraper_name] = scraper_status
                
            except CompetitiveIntelligenceError as e:
                status['scrapers'][scraper_name] = {
                    'error': str(e),
                    'error_type': type(e).__name__,
                    'scraper_configured': False,
                    'scraper_type': 'youtube' if scraper_name.startswith('youtube_') else 'instagram'
                }
            except Exception as e:
                status['scrapers'][scraper_name] = {
                    'error': str(e),
                    'error_type': 'UnexpectedError',
                    'scraper_configured': False,
                    'scraper_type': 'youtube' if scraper_name.startswith('youtube_') else 'instagram'
                }
        
        return status

    def list_available_competitors(self) -> Dict[str, any]:
        """List all available competitors by platform."""
        competitors = {
            'total_scrapers': len(self.scrapers),
            'by_platform': {
                'hvacrschool': ['hvacrschool'],
                'youtube': [],
                'instagram': []
            },
            'all_scrapers': list(self.scrapers.keys())
        }
        
        for scraper_name in self.scrapers.keys():
            if scraper_name.startswith('youtube_'):
                competitors['by_platform']['youtube'].append(scraper_name)
            elif scraper_name.startswith('instagram_'):
                competitors['by_platform']['instagram'].append(scraper_name)
        
        return competitors