import os
import time
import random
import json
import urllib.request
import urllib.parse
from typing import Any, Dict, List, Optional
from datetime import datetime
from pathlib import Path
import yt_dlp
from src.base_scraper import BaseScraper, ScraperConfig
from src.youtube_auth_handler import YouTubeAuthHandler


class YouTubeScraper(BaseScraper):
    """YouTube channel scraper using yt-dlp."""
    
    def __init__(self, config: ScraperConfig):
        super().__init__(config)
        self.channel_url = os.getenv('YOUTUBE_CHANNEL_URL', 'https://www.youtube.com/@HVACKnowItAll')
        # Use videos tab URL to get individual videos instead of playlists
        self.videos_url = self.channel_url.rstrip('/') + '/videos'
        
        # Initialize authentication handler
        self.auth_handler = YouTubeAuthHandler()
        
        # Setup cookies_file attribute for compatibility
        self.cookies_file = Path(config.data_dir) / '.cookies' / 'youtube_cookies.txt'
        self.cookies_file.parent.mkdir(parents=True, exist_ok=True)
        
        # Test authentication on startup
        auth_status = self.auth_handler.get_status()
        if not auth_status['has_valid_cookies']:
            self.logger.warning("No valid YouTube cookies found")
            # Try to extract from browser
            if self.auth_handler.update_cookies_from_browser():
                self.logger.info("Successfully extracted cookies from browser")
            else:
                self.logger.error("Failed to get YouTube authentication")

    def _get_ydl_options(self, include_transcripts: bool = False) -> Dict[str, Any]:
        """Get yt-dlp options with authentication and rate limiting."""
        # Use the auth handler's optimized options
        options = self.auth_handler.get_ytdlp_options(include_auth=True)
        
        # Add transcript options if requested
        if include_transcripts:
            options.update({
                'writesubtitles': True,
                'writeautomaticsub': True,
                'subtitleslangs': ['en'],
            })
        
        # Override with more conservative settings for channel scraping
        options.update({
            'extract_flat': False,  # Get full video info
            'sleep_interval_requests': 20,  # Even more conservative for channel scraping
        })
        
        # Add proxy if configured
        proxy = os.getenv('YOUTUBE_PROXY')
        if proxy:
            options['proxy'] = proxy
        
        return options

    def _humanized_delay(self, min_seconds: float = 2, max_seconds: float = 5) -> None:
        """Add humanized random delay between requests."""
        delay = random.uniform(min_seconds, max_seconds)
        self.logger.debug(f"Waiting {delay:.2f} seconds...")
        time.sleep(delay)
    
    def _backlog_delay(self, transcript_mode: bool = False) -> None:
        """Minimal delay for backlog processing - yt-dlp handles most rate limiting."""
        if transcript_mode:
            # Minimal delay for transcript fetching - let yt-dlp handle it
            base_delay = random.uniform(2, 5)
        else:
            # Minimal delay for basic video info
            base_delay = random.uniform(1, 3)
        
        # Add some randomization to appear more human
        jitter = random.uniform(0.8, 1.2)
        final_delay = base_delay * jitter
        
        self.logger.debug(f"Minimal backlog delay: {final_delay:.1f} seconds...")
        time.sleep(final_delay)

    def fetch_channel_videos(self, max_videos: int = 50) -> List[Dict[str, Any]]:
        """Fetch video list from YouTube channel using auth handler."""
        videos = []
        
        try:
            self.logger.info(f"Fetching videos from channel: {self.videos_url}")
            
            # Use auth handler's optimized extraction with proper cookie management
            ydl_opts = self.auth_handler.get_ytdlp_options(include_auth=True)
            ydl_opts.update({
                'extract_flat': True,  # Just get video list, not full info
                'playlistend': max_videos,
                'sleep_interval_requests': 10,  # Conservative for channel listing
            })
            
            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                channel_info = ydl.extract_info(self.videos_url, download=False)
                
                if 'entries' in channel_info:
                    # Filter out None entries and get actual videos
                    videos = [v for v in channel_info['entries'] if v is not None]
                    self.logger.info(f"Found {len(videos)} videos in channel")
                else:
                    self.logger.warning("No entries found in channel info")
                
        except Exception as e:
            self.logger.error(f"Error fetching channel videos: {e}")
            # Check for bot detection and try recovery
            if self.auth_handler.is_bot_detection_error(str(e)):
                self.logger.warning("Bot detection in channel fetch - attempting recovery")
                self.auth_handler.record_failure(str(e))
                # Try browser cookie update
                if self.auth_handler.update_cookies_from_browser():
                    self.logger.info("Cookie update successful - could retry channel fetch")
        
        return videos

    def fetch_video_details(self, video_id: str, fetch_transcript: bool = False) -> Optional[Dict[str, Any]]:
        """Fetch detailed information for a specific video, optionally including transcript."""
        try:
            video_url = f"https://www.youtube.com/watch?v={video_id}"
            
            # Use auth handler for authenticated extraction with compendium retry logic
            video_info = self.auth_handler.extract_video_info(video_url, max_retries=3)
            
            if not video_info:
                self.logger.error(f"Failed to extract video info for {video_id}")
                
                # If extraction failed, try to update cookies from browser (compendium approach)
                if self.auth_handler.failure_count >= 3:
                    self.logger.warning("Multiple failures detected - attempting browser cookie extraction")
                    if self.auth_handler.update_cookies_from_browser():
                        self.logger.info("Cookie update successful - retrying video extraction")
                        video_info = self.auth_handler.extract_video_info(video_url, max_retries=1)
                
                if not video_info:
                    return None
            
            # Extract transcript if requested and available
            if fetch_transcript:
                transcript = self._extract_transcript(video_info)
                if transcript:
                    video_info['transcript'] = transcript
                    self.logger.info(f"Extracted transcript for video {video_id} ({len(transcript)} chars)")
                else:
                    video_info['transcript'] = None
                    self.logger.warning(f"No transcript available for video {video_id}")
            
            return video_info
                
        except Exception as e:
            self.logger.error(f"Error fetching video {video_id}: {e}")
            # Check if this is a bot detection error and handle accordingly
            if self.auth_handler.is_bot_detection_error(str(e)):
                self.logger.warning("Bot detection error - triggering enhanced recovery")
                self.auth_handler.record_failure(str(e))
                
                # Try browser cookie extraction immediately for bot detection
                if self.auth_handler.update_cookies_from_browser():
                    self.logger.info("Emergency cookie update successful - attempting retry")
                    try:
                        video_info = self.auth_handler.extract_video_info(video_url, max_retries=1)
                        if video_info:
                            if fetch_transcript:
                                transcript = self._extract_transcript(video_info)
                                if transcript:
                                    video_info['transcript'] = transcript
                            return video_info
                    except Exception as retry_error:
                        self.logger.error(f"Retry after cookie update failed: {retry_error}")
            
            return None
    
    def _extract_transcript(self, video_info: Dict[str, Any]) -> Optional[str]:
        """Extract transcript text from video info."""
        try:
            # Try to get subtitles or automatic captions
            subtitles = video_info.get('subtitles', {})
            auto_captions = video_info.get('automatic_captions', {})
            
            # Prefer English subtitles/captions
            transcript_data = None
            transcript_source = None
            
            if 'en' in subtitles:
                transcript_data = subtitles['en']
                transcript_source = "manual subtitles"
            elif 'en' in auto_captions:
                transcript_data = auto_captions['en']
                transcript_source = "auto-generated captions"
            
            if not transcript_data:
                return None
            
            self.logger.debug(f"Using {transcript_source} for video {video_info.get('id')}")
            
            # Find the best format (prefer json3, then srv1, then vtt)
            caption_url = None
            format_preference = ['json3', 'srv1', 'vtt', 'ttml']
            
            for preferred_format in format_preference:
                for caption in transcript_data:
                    if caption.get('ext') == preferred_format:
                        caption_url = caption.get('url')
                        break
                if caption_url:
                    break
            
            if not caption_url:
                # Fallback to first available format
                if transcript_data:
                    caption_url = transcript_data[0].get('url')
            
            if not caption_url:
                return None
            
            # Fetch and parse the transcript
            return self._fetch_and_parse_transcript(caption_url, video_info.get('id'))
                
        except Exception as e:
            self.logger.error(f"Error extracting transcript: {e}")
            return None
    
    def _fetch_and_parse_transcript(self, caption_url: str, video_id: str) -> Optional[str]:
        """Fetch and parse transcript from caption URL."""
        try:
            # Fetch the caption content
            with urllib.request.urlopen(caption_url) as response:
                content = response.read().decode('utf-8')
            
            # Parse based on format
            if 'json3' in caption_url or caption_url.endswith('.json'):
                return self._parse_json_transcript(content)
            elif 'srv1' in caption_url or 'srv2' in caption_url:
                return self._parse_srv_transcript(content)
            elif caption_url.endswith('.vtt'):
                return self._parse_vtt_transcript(content)
            else:
                # Try to auto-detect format
                content_lower = content.lower().strip()
                if content_lower.startswith('{') or 'wiremag' in content_lower:
                    return self._parse_json_transcript(content)
                elif 'webvtt' in content_lower:
                    return self._parse_vtt_transcript(content)
                elif '<transcript>' in content_lower or '<text>' in content_lower:
                    return self._parse_srv_transcript(content)
                else:
                    # Last resort - return raw content
                    self.logger.warning(f"Unknown transcript format for {video_id}, returning raw content")
                    return content
                    
        except Exception as e:
            self.logger.error(f"Error fetching transcript for video {video_id}: {e}")
            return None
    
    def _parse_json_transcript(self, content: str) -> Optional[str]:
        """Parse JSON3 format transcript."""
        try:
            data = json.loads(content)
            transcript_parts = []
            
            # Handle YouTube's JSON3 format
            if 'events' in data:
                for event in data['events']:
                    if 'segs' in event:
                        for seg in event['segs']:
                            if 'utf8' in seg:
                                text = seg['utf8'].strip()
                                if text and text not in ['♪', '[Music]', '[Applause]']:
                                    transcript_parts.append(text)
            
            return ' '.join(transcript_parts) if transcript_parts else None
            
        except Exception as e:
            self.logger.error(f"Error parsing JSON transcript: {e}")
            return None
    
    def _parse_srv_transcript(self, content: str) -> Optional[str]:
        """Parse SRV format transcript (XML-like)."""
        try:
            import xml.etree.ElementTree as ET
            
            # Parse XML content
            root = ET.fromstring(content)
            transcript_parts = []
            
            # Extract text from <text> elements
            for text_elem in root.findall('.//text'):
                text = text_elem.text
                if text and text.strip():
                    clean_text = text.strip()
                    if clean_text not in ['♪', '[Music]', '[Applause]']:
                        transcript_parts.append(clean_text)
            
            return ' '.join(transcript_parts) if transcript_parts else None
            
        except Exception as e:
            self.logger.error(f"Error parsing SRV transcript: {e}")
            return None
    
    def _parse_vtt_transcript(self, content: str) -> Optional[str]:
        """Parse VTT format transcript."""
        try:
            lines = content.split('\n')
            transcript_parts = []
            
            for line in lines:
                line = line.strip()
                # Skip VTT headers, timestamps, and empty lines
                if (not line or 
                    line.startswith('WEBVTT') or 
                    line.startswith('NOTE') or 
                    '-->' in line or 
                    line.isdigit()):
                    continue
                
                # Clean up common caption artifacts
                if line not in ['♪', '[Music]', '[Applause]', '&nbsp;']:
                    # Remove HTML tags if present
                    import re
                    clean_line = re.sub(r'<[^>]+>', '', line)
                    if clean_line.strip():
                        transcript_parts.append(clean_line.strip())
            
            return ' '.join(transcript_parts) if transcript_parts else None
            
        except Exception as e:
            self.logger.error(f"Error parsing VTT transcript: {e}")
            return None

    def _get_video_type(self, video: Dict[str, Any]) -> str:
        """Determine video type (video, short, live)."""
        duration = video.get('duration', 0)
        is_live = video.get('is_live', False)
        
        if is_live:
            return 'live'
        elif duration and duration < 60:  # Less than 60 seconds
            return 'short'
        else:
            return 'video'

    def fetch_content(self, max_posts: int = None, fetch_transcripts: bool = False) -> List[Dict[str, Any]]:
        """Fetch and enrich video content with rate limiting."""
        # First get list of videos
        videos = self.fetch_channel_videos()
        
        if not videos:
            return []
        
        # Limit videos if max_posts specified
        if max_posts:
            videos = videos[:max_posts]
        
        # Enrich each video with detailed information
        enriched_videos = []
        
        for i, video in enumerate(videos):
            try:
                video_id = video.get('id')
                if not video_id:
                    continue
                
                transcript_note = " (with transcripts)" if fetch_transcripts else ""
                self.logger.info(f"Fetching details for video {i+1}/{len(videos)}: {video_id}{transcript_note}")
                
                # Determine if this is backlog processing (no max_posts = full backlog)
                is_backlog = max_posts is None
                
                # Add appropriate delay between requests
                if i > 0:
                    if is_backlog:
                        # Use extended backlog delays (30-90 seconds for transcripts)
                        self._backlog_delay(transcript_mode=fetch_transcripts)
                    else:
                        # Use normal delays for limited fetching
                        self._humanized_delay()
                
                # Fetch full video details with optional transcripts
                detailed_info = self.fetch_video_details(video_id, fetch_transcript=fetch_transcripts)
                
                if detailed_info:
                    # Add video type
                    detailed_info['type'] = self._get_video_type(detailed_info)
                    enriched_videos.append(detailed_info)
                    
                    # Extra delay after every 5 videos for backlog processing
                    if is_backlog and (i + 1) % 5 == 0:
                        self.logger.info("Taking extended break after 5 videos (backlog mode)...")
                        # Even longer break every 5 videos for backlog (2-5 minutes)
                        extra_delay = random.uniform(120, 300)  # 2-5 minutes
                        self.logger.info(f"Extended break: {extra_delay/60:.1f} minutes...")
                        time.sleep(extra_delay)
                else:
                    # If video details failed and we're doing transcripts, check for rate limiting
                    if fetch_transcripts and is_backlog:
                        self.logger.warning(f"Failed to get details for video {video_id} - may be rate limited")
                        # Add emergency rate limiting delay
                        emergency_delay = random.uniform(180, 300)  # 3-5 minutes
                        self.logger.info(f"Emergency rate limit delay: {emergency_delay/60:.1f} minutes...")
                        time.sleep(emergency_delay)
                
            except Exception as e:
                self.logger.error(f"Error enriching video {video.get('id')}: {e}")
                continue
        
        self.logger.info(f"Successfully enriched {len(enriched_videos)} videos")
        return enriched_videos

    def format_markdown(self, videos: List[Dict[str, Any]]) -> str:
        """Format videos as markdown."""
        markdown_sections = []
        
        for video in videos:
            section = []
            
            # ID
            video_id = video.get('id', 'N/A')
            section.append(f"# ID: {video_id}")
            section.append("")
            
            # Title
            title = video.get('title', 'Untitled')
            section.append(f"## Title: {title}")
            section.append("")
            
            # Type
            video_type = video.get('type', self._get_video_type(video))
            section.append(f"## Type: {video_type}")
            section.append("")
            
            # Author/Uploader
            author = video.get('uploader', 'Unknown')
            section.append(f"## Author: {author}")
            section.append("")
            
            # Link
            link = video.get('webpage_url', f"https://www.youtube.com/watch?v={video_id}")
            section.append(f"## Link: {link}")
            section.append("")
            
            # Upload Date
            upload_date = video.get('upload_date', '')
            if upload_date and len(upload_date) == 8:  # YYYYMMDD format
                formatted_date = f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:]}"
                section.append(f"## Upload Date: {formatted_date}")
            else:
                section.append(f"## Upload Date: {upload_date}")
            section.append("")
            
            # Views
            view_count = video.get('view_count', 0)
            section.append(f"## Views: {view_count}")
            section.append("")
            
            # Likes
            like_count = video.get('like_count', 0)
            section.append(f"## Likes: {like_count}")
            section.append("")
            
            # Comments
            comment_count = video.get('comment_count', 0)
            section.append(f"## Comments: {comment_count}")
            section.append("")
            
            # Duration
            duration = video.get('duration', 0)
            section.append(f"## Duration: {duration} seconds")
            section.append("")
            
            # Tags
            tags = video.get('tags', [])
            if tags:
                tags_str = ', '.join(tags[:10])  # Limit to first 10 tags
                section.append(f"## Tags: {tags_str}")
                section.append("")
            
            # Thumbnail
            thumbnail = video.get('thumbnail', '')
            if thumbnail:
                section.append(f"## Thumbnail: {thumbnail}")
                section.append("")
            
            # Description
            section.append("## Description:")
            description = video.get('description', '')
            if description:
                # Limit description to first 500 characters
                if len(description) > 500:
                    description = description[:500] + "..."
                section.append(description)
            section.append("")
            
            # Transcript
            transcript = video.get('transcript')
            if transcript:
                section.append("## Transcript:")
                section.append(transcript)
                section.append("")
            
            # Separator
            section.append("-" * 50)
            section.append("")
            
            markdown_sections.append('\n'.join(section))
        
        return '\n'.join(markdown_sections)

    def get_incremental_items(self, items: List[Dict[str, Any]], state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Get only new videos since last sync."""
        if not state:
            return items
        
        last_video_id = state.get('last_video_id')
        last_video_date = state.get('last_video_date')
        
        if not last_video_id:
            return items
        
        # Filter for videos newer than the last synced
        new_items = []
        for item in items:
            video_id = item.get('id')
            upload_date = item.get('upload_date', '')
            
            # Check if this is a new video
            if video_id == last_video_id:
                break  # Found the last synced video, stop here
            
            # Also check by date as backup
            if upload_date and last_video_date and upload_date <= last_video_date:
                continue
            
            new_items.append(item)
        
        return new_items

    def update_state(self, state: Dict[str, Any], items: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Update state with latest video information."""
        if not items:
            return state
        
        # Get the first item (most recent)
        latest_item = items[0]
        
        state['last_video_id'] = latest_item.get('id')
        state['last_video_date'] = latest_item.get('upload_date')
        state['last_video_title'] = latest_item.get('title')
        state['last_sync'] = datetime.now(self.tz).isoformat()
        state['video_count'] = len(items)
        
        return state