#!/usr/bin/env python3
"""
TRUE HYBRID YouTube Scraper
- YouTube Data API v3 for metadata (cheap, reliable)
- yt-dlp with authentication for transcripts only (when not blocked)
"""

import os
import time
from typing import Any, Dict, List, Optional
from datetime import datetime
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
import yt_dlp
from src.base_scraper import BaseScraper, ScraperConfig
from src.youtube_auth_handler import YouTubeAuthHandler
from src.youtube_po_token_handler import YouTubePOTokenHandler


class YouTubeHybridScraper(BaseScraper):
    """True hybrid YouTube scraper: API for metadata, yt-dlp for transcripts."""
    
    def __init__(self, config: ScraperConfig):
        super().__init__(config)
        
        # YouTube Data API v3 setup
        self.api_key = os.getenv('YOUTUBE_API_KEY')
        if not self.api_key:
            raise ValueError("YOUTUBE_API_KEY not found in environment variables")
        
        self.youtube = build('youtube', 'v3', developerKey=self.api_key)
        
        # Channel configuration
        self.channel_url = os.getenv('YOUTUBE_CHANNEL_URL', 'https://www.youtube.com/@HVACKnowItAll')
        self.channel_id = None
        self.uploads_playlist_id = None
        
        # Quota tracking for API
        self.quota_used = 0
        self.daily_quota_limit = 10000
        
        # yt-dlp setup for transcripts with PO token support
        self.auth_handler = YouTubeAuthHandler()
        self.po_token_handler = YouTubePOTokenHandler(logger=self.logger)
        
        # Test authentication on startup
        auth_status = self.auth_handler.get_status()
        po_status = self.po_token_handler.get_status()
        
        self.logger.info(f"Firefox profile found: {po_status['firefox_profile_found']}")
        self.logger.info(f"Environment PO tokens: {len(po_status['env_tokens_available'])}")
        
        if not auth_status['has_valid_cookies']:
            self.logger.warning("No valid YouTube cookies found")
            if self.auth_handler.update_cookies_from_browser():
                self.logger.info("Successfully extracted cookies from browser")
            else:
                self.logger.warning("Failed to get YouTube authentication")
        
        self.logger.info(f"Hybrid scraper initialized for channel: {self.channel_url}")
    
    def _track_quota(self, operation: str, count: int = 1) -> bool:
        """Track API quota usage."""
        costs = {'channels_list': 1, 'playlist_items': 1, 'videos_list': 1}
        cost = costs.get(operation, 0) * count
        
        if self.quota_used + cost > self.daily_quota_limit:
            self.logger.warning(f"API quota limit would be exceeded")
            return False
        
        self.quota_used += cost
        return True
    
    def _get_channel_info(self) -> bool:
        """Get channel info using YouTube Data API."""
        if self.channel_id and self.uploads_playlist_id:
            return True
        
        try:
            channel_handle = self.channel_url.split('@')[-1]
            
            if not self._track_quota('channels_list'):
                return False
            
            response = self.youtube.channels().list(
                part='snippet,statistics,contentDetails',
                forHandle=channel_handle
            ).execute()
            
            if response.get('items'):
                channel_data = response['items'][0]
                self.channel_id = channel_data['id']
                self.uploads_playlist_id = channel_data['contentDetails']['relatedPlaylists']['uploads']
                
                stats = channel_data['statistics']
                self.logger.info(f"Channel: {channel_data['snippet']['title']}")
                self.logger.info(f"Subscribers: {int(stats.get('subscriberCount', 0)):,}")
                self.logger.info(f"Total videos: {int(stats.get('videoCount', 0)):,}")
                return True
            
        except HttpError as e:
            self.logger.error(f"YouTube API error: {e}")
        except Exception as e:
            self.logger.error(f"Error getting channel info: {e}")
        
        return False
    
    def _fetch_video_ids_api(self, max_videos: int = None) -> List[str]:
        """Fetch video IDs using YouTube Data API (cheap)."""
        if not self._get_channel_info():
            return []
        
        video_ids = []
        next_page_token = None
        videos_fetched = 0
        
        while True:
            if not self._track_quota('playlist_items'):
                break
            
            try:
                response = self.youtube.playlistItems().list(
                    part='contentDetails',
                    playlistId=self.uploads_playlist_id,
                    maxResults=50,
                    pageToken=next_page_token
                ).execute()
                
                for item in response.get('items', []):
                    video_ids.append(item['contentDetails']['videoId'])
                    videos_fetched += 1
                    
                    if max_videos and videos_fetched >= max_videos:
                        return video_ids[:max_videos]
                
                next_page_token = response.get('nextPageToken')
                if not next_page_token:
                    break
                    
            except HttpError as e:
                self.logger.error(f"Error fetching video IDs: {e}")
                break
        
        self.logger.info(f"Fetched {len(video_ids)} video IDs using API")
        return video_ids
    
    def _fetch_video_details_api(self, video_ids: List[str]) -> List[Dict[str, Any]]:
        """Fetch video metadata using YouTube Data API (cheap)."""
        if not video_ids:
            return []
        
        batch_size = 50
        all_videos = []
        
        for i in range(0, len(video_ids), batch_size):
            batch = video_ids[i:i + batch_size]
            
            if not self._track_quota('videos_list'):
                break
            
            try:
                response = self.youtube.videos().list(
                    part='snippet,statistics,contentDetails',
                    id=','.join(batch)
                ).execute()
                
                for video in response.get('items', []):
                    video_data = {
                        'id': video['id'],
                        'title': video['snippet']['title'],
                        'description': video['snippet']['description'],  # Full description!
                        'published_at': video['snippet']['publishedAt'],
                        'channel_title': video['snippet']['channelTitle'],
                        'tags': video['snippet'].get('tags', []),
                        'duration': video['contentDetails']['duration'],
                        'thumbnail': video['snippet']['thumbnails'].get('maxres', {}).get('url') or 
                                    video['snippet']['thumbnails'].get('high', {}).get('url', ''),
                        
                        # Rich statistics from API
                        'view_count': int(video['statistics'].get('viewCount', 0)),
                        'like_count': int(video['statistics'].get('likeCount', 0)),
                        'comment_count': int(video['statistics'].get('commentCount', 0)),
                        'engagement_rate': 0,
                    }
                    
                    # Calculate engagement
                    if video_data['view_count'] > 0:
                        video_data['engagement_rate'] = (
                            (video_data['like_count'] + video_data['comment_count']) / 
                            video_data['view_count']
                        ) * 100
                    
                    all_videos.append(video_data)
                
                time.sleep(0.1)  # Be respectful
                
            except HttpError as e:
                self.logger.error(f"Error fetching video details: {e}")
        
        return all_videos
    
    def _fetch_transcript_ytdlp(self, video_id: str) -> Optional[str]:
        """Fetch transcript using yt-dlp with PO token support (true hybrid approach)."""
        
        # First try the PO token handler method (modern approach)
        transcript = self.po_token_handler.extract_subtitle_with_token(video_id)
        if transcript:
            self.logger.debug(f"Successfully extracted transcript using PO token for {video_id}")
            return transcript
        
        # Fallback to legacy auth handler method 
        try:
            video_url = f"https://www.youtube.com/watch?v={video_id}"
            
            # Use auth handler for authenticated extraction (fallback)
            video_info = self.auth_handler.extract_video_info(video_url, max_retries=3)
            
            if not video_info:
                return None
            
            # Extract transcript using the same logic as original YouTube scraper
            subtitles = video_info.get('subtitles', {})
            auto_captions = video_info.get('automatic_captions', {})
            
            transcript_data = None
            if 'en' in subtitles:
                transcript_data = subtitles['en']
            elif 'en' in auto_captions:
                transcript_data = auto_captions['en']
            
            if not transcript_data:
                return None
            
            # Get caption URL
            caption_url = None
            for caption in transcript_data:
                if caption.get('ext') in ['json3', 'srv1', 'vtt']:
                    caption_url = caption.get('url')
                    break
            
            if not caption_url and transcript_data:
                caption_url = transcript_data[0].get('url')
            
            if caption_url:
                # Fetch and parse transcript
                import urllib.request
                with urllib.request.urlopen(caption_url) as response:
                    content = response.read().decode('utf-8')
                
                # Simple parsing - extract text
                if 'json3' in caption_url:
                    import json
                    data = json.loads(content)
                    transcript_parts = []
                    if 'events' in data:
                        for event in data['events']:
                            if 'segs' in event:
                                for seg in event['segs']:
                                    if 'utf8' in seg:
                                        text = seg['utf8'].strip()
                                        if text and text not in ['♪', '[Music]']:
                                            transcript_parts.append(text)
                    return ' '.join(transcript_parts)
                
                return content  # Fallback to raw content
                
        except Exception as e:
            self.logger.debug(f"Legacy transcript extraction failed for {video_id}: {e}")
            return None
    
    def fetch_content(self, max_posts: int = None, fetch_transcripts: bool = False) -> List[Dict[str, Any]]:
        """Hybrid approach: API for metadata, yt-dlp for transcripts."""
        
        self.logger.info(f"Starting hybrid YouTube fetch")
        start_time = time.time()
        
        # Step 1: Get video IDs using API (very cheap)
        video_ids = self._fetch_video_ids_api(max_posts)
        if not video_ids:
            return []
        
        # Step 2: Get video metadata using API (cheap, rich data)
        videos = self._fetch_video_details_api(video_ids)
        
        api_time = time.time() - start_time
        self.logger.info(f"API phase: {len(videos)} videos in {api_time:.1f}s (quota: {self.quota_used})")
        
        # Step 3: Get transcripts using yt-dlp with auth (when requested)
        if fetch_transcripts and videos:
            # Prioritize by views for transcript fetching
            videos_sorted = sorted(videos, key=lambda x: x['view_count'], reverse=True)
            max_transcripts = min(10, len(videos_sorted))  # Limit to top 10 for testing
            
            self.logger.info(f"Fetching transcripts for top {max_transcripts} videos using yt-dlp")
            
            transcript_start = time.time()
            for i, video in enumerate(videos_sorted[:max_transcripts]):
                transcript = self._fetch_transcript_ytdlp(video['id'])
                if transcript:
                    video['transcript'] = transcript
                    self.logger.info(f"Got transcript {i+1}/{max_transcripts}: {video['title'][:50]}...")
                else:
                    video['transcript'] = None
                
                # Rate limiting for yt-dlp requests
                if i < max_transcripts - 1:
                    time.sleep(2)
            
            transcript_time = time.time() - transcript_start
            with_transcripts = sum(1 for v in videos if v.get('transcript'))
            self.logger.info(f"Transcript phase: {with_transcripts}/{max_transcripts} in {transcript_time:.1f}s")
        
        total_time = time.time() - start_time
        self.logger.info(f"Hybrid fetch complete: {len(videos)} videos in {total_time:.1f}s")
        self.logger.info(f"API quota used: {self.quota_used}/{self.daily_quota_limit}")
        
        return videos
    
    def _get_video_type(self, video: Dict[str, Any]) -> str:
        """Determine video type based on duration."""
        duration = video.get('duration', 'PT0S')
        
        import re
        match = re.match(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?', duration)
        if match:
            hours = int(match.group(1) or 0)
            minutes = int(match.group(2) or 0) 
            seconds = int(match.group(3) or 0)
            total_seconds = hours * 3600 + minutes * 60 + seconds
            
            if total_seconds < 60:
                return 'short'
            else:
                return 'video'
        
        return 'video'
    
    def format_markdown(self, videos: List[Dict[str, Any]]) -> str:
        """Format videos as markdown with hybrid data."""
        markdown_sections = []
        
        for video in videos:
            section = []
            
            section.append(f"# ID: {video.get('id', 'N/A')}")
            section.append("")
            section.append(f"## Title: {video.get('title', 'Untitled')}")
            section.append("")
            section.append(f"## Type: {self._get_video_type(video)}")
            section.append("")
            section.append(f"## Author: {video.get('channel_title', 'Unknown')}")
            section.append("")
            section.append(f"## Link: https://www.youtube.com/watch?v={video.get('id')}")
            section.append("")
            section.append(f"## Upload Date: {video.get('published_at', '')}")
            section.append("")
            section.append(f"## Duration: {video.get('duration', 'Unknown')}")
            section.append("")
            section.append(f"## Views: {video.get('view_count', 0):,}")
            section.append("")
            section.append(f"## Likes: {video.get('like_count', 0):,}")
            section.append("")
            section.append(f"## Comments: {video.get('comment_count', 0):,}")
            section.append("")
            section.append(f"## Engagement Rate: {video.get('engagement_rate', 0):.2f}%")
            section.append("")
            
            # Tags
            tags = video.get('tags', [])
            if tags:
                section.append(f"## Tags: {', '.join(tags[:10])}")
                section.append("")
            
            # Thumbnail
            thumbnail = video.get('thumbnail', '')
            if thumbnail:
                section.append(f"## Thumbnail: {thumbnail}")
                section.append("")
            
            # Full Description
            section.append("## Description:")
            description = video.get('description', '')
            if description:
                section.append(description)
            section.append("")
            
            # Transcript (from yt-dlp)
            transcript = video.get('transcript')
            if transcript:
                section.append("## Transcript:")
                section.append(transcript)
                section.append("")
            
            section.append("-" * 50)
            section.append("")
            
            markdown_sections.append('\n'.join(section))
        
        return '\n'.join(markdown_sections)
    
    def get_incremental_items(self, items: List[Dict[str, Any]], state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Get only new videos since last sync."""
        if not state:
            return items
        
        last_video_id = state.get('last_video_id')
        if not last_video_id:
            return items
        
        # Filter for videos newer than the last synced
        new_items = []
        for item in items:
            if item.get('id') == last_video_id:
                break
            new_items.append(item)
        
        return new_items
    
    def update_state(self, state: Dict[str, Any], items: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Update state with latest video information."""
        if not items:
            return state
        
        latest_item = items[0]
        state['last_video_id'] = latest_item.get('id')
        state['last_published'] = latest_item.get('published_at')
        state['last_video_title'] = latest_item.get('title')
        state['last_sync'] = datetime.now(self.tz).isoformat()
        state['video_count'] = len(items)
        state['quota_used'] = self.quota_used
        
        return state