#!/usr/bin/env python3
"""
YouTube Data API v3 scraper with quota management and captions support
Designed to stay within 10,000 units/day limit while fetching captions

Quota costs:
- channels.list: 1 unit
- playlistItems.list: 1 unit per page (50 items max)
- videos.list: 1 unit per page (50 videos max)
- search.list: 100 units (avoid if possible!)
- captions.list: 50 units per video
- captions.download: 200 units per caption

Strategy for 444 videos with captions:
- Get channel info: 1 unit
- Get all playlist items (444/50 = 9 pages): 9 units
- Get video details in batches of 50: 9 units
- Get captions list for each video: 444 * 50 = 22,200 units (too much!)
- Alternative: Use captions.list selectively or in batches
"""

import os
import time
from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from src.base_scraper import BaseScraper, ScraperConfig
import logging
import re


class YouTubeAPIScraper(BaseScraper):
    """YouTube API scraper with quota management and captions."""
    
    # Quota costs for different operations
    QUOTA_COSTS = {
        'channels_list': 1,
        'playlist_items': 1,
        'videos_list': 1,
        'search': 100,
        'captions_list': 50,
        'captions_download': 200,
    }
    
    def __init__(self, config: ScraperConfig):
        super().__init__(config)
        
        self.api_key = os.getenv('YOUTUBE_API_KEY')
        if not self.api_key:
            raise ValueError("YOUTUBE_API_KEY not found in environment variables")
        
        # Build YouTube API client
        self.youtube = build('youtube', 'v3', developerKey=self.api_key)
        
        # Channel configuration
        self.channel_url = os.getenv('YOUTUBE_CHANNEL_URL', 'https://www.youtube.com/@HVACKnowItAll')
        self.channel_id = None
        self.uploads_playlist_id = None
        
        # Quota tracking
        self.quota_used = 0
        self.daily_quota_limit = 10000
        
        # Caption fetching strategy
        self.max_captions_per_run = 50  # Limit caption fetches to top videos
        # 50 videos * 50 units = 2,500 units for caption listing
        # Plus potential download costs
        
        self.logger.info(f"Initialized YouTube API scraper for channel: {self.channel_url}")
    
    def _track_quota(self, operation: str, count: int = 1) -> bool:
        """Track quota usage and return True if within limits."""
        cost = self.QUOTA_COSTS.get(operation, 0) * count
        
        if self.quota_used + cost > self.daily_quota_limit:
            self.logger.warning(f"Quota limit would be exceeded. Current: {self.quota_used}, Cost: {cost}")
            return False
        
        self.quota_used += cost
        self.logger.debug(f"Quota used: {self.quota_used}/{self.daily_quota_limit} (+{cost} for {operation})")
        return True
    
    def _get_channel_info(self) -> bool:
        """Get channel ID and uploads playlist ID."""
        if self.channel_id and self.uploads_playlist_id:
            return True
        
        try:
            # Extract channel handle
            channel_handle = self.channel_url.split('@')[-1]
            
            # Try to get channel by handle first (costs 1 unit)
            if not self._track_quota('channels_list'):
                return False
            
            response = self.youtube.channels().list(
                part='snippet,statistics,contentDetails',
                forHandle=channel_handle
            ).execute()
            
            if not response.get('items'):
                # Fallback to search by name (costs 100 units - avoid!)
                self.logger.warning("Channel not found by handle, trying search...")
                
                if not self._track_quota('search'):
                    return False
                
                search_response = self.youtube.search().list(
                    part='snippet',
                    q="HVAC Know It All",
                    type='channel',
                    maxResults=1
                ).execute()
                
                if not search_response.get('items'):
                    self.logger.error("Channel not found")
                    return False
                
                self.channel_id = search_response['items'][0]['snippet']['channelId']
                
                # Get full channel details
                if not self._track_quota('channels_list'):
                    return False
                
                response = self.youtube.channels().list(
                    part='snippet,statistics,contentDetails',
                    id=self.channel_id
                ).execute()
            
            if response.get('items'):
                channel_data = response['items'][0]
                self.channel_id = channel_data['id']
                self.uploads_playlist_id = channel_data['contentDetails']['relatedPlaylists']['uploads']
                
                # Log channel stats
                stats = channel_data['statistics']
                self.logger.info(f"Channel: {channel_data['snippet']['title']}")
                self.logger.info(f"Subscribers: {int(stats.get('subscriberCount', 0)):,}")
                self.logger.info(f"Total videos: {int(stats.get('videoCount', 0)):,}")
                
                return True
            
        except HttpError as e:
            self.logger.error(f"YouTube API error: {e}")
        except Exception as e:
            self.logger.error(f"Error getting channel info: {e}")
        
        return False
    
    def _fetch_all_video_ids(self, max_videos: int = None) -> List[str]:
        """Fetch all video IDs from the channel efficiently."""
        if not self._get_channel_info():
            return []
        
        video_ids = []
        next_page_token = None
        videos_fetched = 0
        
        while True:
            # Check quota before each request
            if not self._track_quota('playlist_items'):
                self.logger.warning("Quota limit reached while fetching video IDs")
                break
            
            try:
                # Fetch playlist items (50 per page, costs 1 unit)
                request = self.youtube.playlistItems().list(
                    part='contentDetails',
                    playlistId=self.uploads_playlist_id,
                    maxResults=50,
                    pageToken=next_page_token
                )
                
                response = request.execute()
                
                for item in response.get('items', []):
                    video_ids.append(item['contentDetails']['videoId'])
                    videos_fetched += 1
                    
                    if max_videos and videos_fetched >= max_videos:
                        return video_ids[:max_videos]
                
                # Check for next page
                next_page_token = response.get('nextPageToken')
                if not next_page_token:
                    break
                
            except HttpError as e:
                self.logger.error(f"Error fetching video IDs: {e}")
                break
        
        self.logger.info(f"Fetched {len(video_ids)} video IDs")
        return video_ids
    
    def _fetch_video_details_batch(self, video_ids: List[str]) -> List[Dict[str, Any]]:
        """Fetch details for a batch of videos (max 50 per request)."""
        if not video_ids:
            return []
        
        # YouTube API allows max 50 videos per request
        batch_size = 50
        all_videos = []
        
        for i in range(0, len(video_ids), batch_size):
            batch = video_ids[i:i + batch_size]
            
            # Check quota (1 unit per request)
            if not self._track_quota('videos_list'):
                self.logger.warning("Quota limit reached while fetching video details")
                break
            
            try:
                response = self.youtube.videos().list(
                    part='snippet,statistics,contentDetails',
                    id=','.join(batch)
                ).execute()
                
                for video in response.get('items', []):
                    video_data = {
                        'id': video['id'],
                        'title': video['snippet']['title'],
                        'description': video['snippet']['description'],  # Full description!
                        'published_at': video['snippet']['publishedAt'],
                        'channel_id': video['snippet']['channelId'],
                        'channel_title': video['snippet']['channelTitle'],
                        'tags': video['snippet'].get('tags', []),
                        'duration': video['contentDetails']['duration'],
                        'definition': video['contentDetails']['definition'],
                        'caption': video['contentDetails'].get('caption', 'false'),  # Has captions?
                        'thumbnail': video['snippet']['thumbnails'].get('maxres', {}).get('url') or 
                                    video['snippet']['thumbnails'].get('high', {}).get('url', ''),
                        
                        # Statistics
                        'view_count': int(video['statistics'].get('viewCount', 0)),
                        'like_count': int(video['statistics'].get('likeCount', 0)),
                        'comment_count': int(video['statistics'].get('commentCount', 0)),
                        
                        # Calculate engagement metrics
                        'engagement_rate': 0,
                        'like_ratio': 0
                    }
                    
                    # Calculate engagement metrics
                    if video_data['view_count'] > 0:
                        video_data['engagement_rate'] = (
                            (video_data['like_count'] + video_data['comment_count']) / 
                            video_data['view_count']
                        ) * 100
                        video_data['like_ratio'] = (video_data['like_count'] / video_data['view_count']) * 100
                    
                    all_videos.append(video_data)
                
                # Small delay to be respectful
                time.sleep(0.1)
                
            except HttpError as e:
                self.logger.error(f"Error fetching video details: {e}")
        
        return all_videos
    
    def _fetch_caption_text(self, video_id: str) -> Optional[str]:
        """Fetch caption text using YouTube Data API (costs 50 units!)."""
        try:
            # Check quota (50 units for list)
            if not self._track_quota('captions_list'):
                self.logger.debug(f"Quota limit - skipping captions for {video_id}")
                return None
            
            # List available captions
            captions_response = self.youtube.captions().list(
                part='snippet',
                videoId=video_id
            ).execute()
            
            captions = captions_response.get('items', [])
            if not captions:
                self.logger.debug(f"No captions available for video {video_id}")
                return None
            
            # Find English caption (or auto-generated)
            english_caption = None
            for caption in captions:
                if caption['snippet']['language'] == 'en':
                    english_caption = caption
                    break
            
            if not english_caption:
                # Try auto-generated
                for caption in captions:
                    if 'auto' in caption['snippet']['name'].lower():
                        english_caption = caption
                        break
            
            if english_caption:
                caption_id = english_caption['id']
                
                # Download caption would cost 200 more units!
                # For now, just note that captions are available
                self.logger.debug(f"Captions available for video {video_id} (id: {caption_id})")
                return f"[Captions available - {english_caption['snippet']['name']}]"
            
            return None
            
        except HttpError as e:
            if 'captionsDisabled' in str(e):
                self.logger.debug(f"Captions disabled for video {video_id}")
            else:
                self.logger.debug(f"Error fetching captions for {video_id}: {e}")
        except Exception as e:
            self.logger.debug(f"Error fetching captions for {video_id}: {e}")
        
        return None
    
    def fetch_content(self, max_posts: int = None, fetch_captions: bool = True) -> List[Dict[str, Any]]:
        """Fetch video content with intelligent quota management."""
        
        self.logger.info(f"Starting YouTube API fetch (quota limit: {self.daily_quota_limit})")
        
        # Step 1: Get all video IDs (very cheap - ~9 units for 444 videos)
        video_ids = self._fetch_all_video_ids(max_posts)
        
        if not video_ids:
            self.logger.warning("No video IDs fetched")
            return []
        
        # Step 2: Fetch video details in batches (also cheap - ~9 units for 444 videos)
        videos = self._fetch_video_details_batch(video_ids)
        
        self.logger.info(f"Fetched details for {len(videos)} videos")
        
        # Step 3: Fetch captions for top videos (expensive - 50 units per video)
        if fetch_captions:
            # Prioritize videos by views for caption fetching
            videos_sorted = sorted(videos, key=lambda x: x['view_count'], reverse=True)
            
            # Limit caption fetching to top videos
            max_captions = min(self.max_captions_per_run, len(videos_sorted))
            
            # Check remaining quota
            captions_quota_needed = max_captions * 50
            if self.quota_used + captions_quota_needed > self.daily_quota_limit:
                max_captions = (self.daily_quota_limit - self.quota_used) // 50
                self.logger.warning(f"Limiting captions to {max_captions} videos due to quota")
            
            if max_captions > 0:
                self.logger.info(f"Fetching captions for top {max_captions} videos by views")
                
                for i, video in enumerate(videos_sorted[:max_captions]):
                    caption_text = self._fetch_caption_text(video['id'])
                    if caption_text:
                        video['caption_text'] = caption_text
                        self.logger.debug(f"Got caption info for video {i+1}/{max_captions}: {video['title']}")
                    
                    # Small delay to be respectful
                    time.sleep(0.5)
        
        # Log final quota usage
        self.logger.info(f"Total quota used: {self.quota_used}/{self.daily_quota_limit} units")
        self.logger.info(f"Remaining quota: {self.daily_quota_limit - self.quota_used} units")
        
        return videos
    
    def _get_video_type(self, video: Dict[str, Any]) -> str:
        """Determine video type based on duration."""
        duration = video.get('duration', 'PT0S')
        
        # Parse ISO 8601 duration
        match = re.match(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?', duration)
        if match:
            hours = int(match.group(1) or 0)
            minutes = int(match.group(2) or 0) 
            seconds = int(match.group(3) or 0)
            total_seconds = hours * 3600 + minutes * 60 + seconds
            
            if total_seconds < 60:
                return 'short'
            elif total_seconds > 600:  # > 10 minutes
                return 'video'
            else:
                return 'video'
        
        return 'video'
    
    def format_markdown(self, videos: List[Dict[str, Any]]) -> str:
        """Format videos as markdown with enhanced data."""
        markdown_sections = []
        
        for video in videos:
            section = []
            
            # ID
            section.append(f"# ID: {video.get('id', 'N/A')}")
            section.append("")
            
            # Title
            section.append(f"## Title: {video.get('title', 'Untitled')}")
            section.append("")
            
            # Type
            video_type = self._get_video_type(video)
            section.append(f"## Type: {video_type}")
            section.append("")
            
            # Author
            section.append(f"## Author: {video.get('channel_title', 'Unknown')}")
            section.append("")
            
            # Link
            section.append(f"## Link: https://www.youtube.com/watch?v={video.get('id')}")
            section.append("")
            
            # Upload Date
            section.append(f"## Upload Date: {video.get('published_at', '')}")
            section.append("")
            
            # Duration
            section.append(f"## Duration: {video.get('duration', 'Unknown')}")
            section.append("")
            
            # Views
            section.append(f"## Views: {video.get('view_count', 0):,}")
            section.append("")
            
            # Likes
            section.append(f"## Likes: {video.get('like_count', 0):,}")
            section.append("")
            
            # Comments
            section.append(f"## Comments: {video.get('comment_count', 0):,}")
            section.append("")
            
            # Engagement Metrics
            section.append(f"## Engagement Rate: {video.get('engagement_rate', 0):.2f}%")
            section.append(f"## Like Ratio: {video.get('like_ratio', 0):.2f}%")
            section.append("")
            
            # Tags
            tags = video.get('tags', [])
            if tags:
                section.append(f"## Tags: {', '.join(tags[:10])}")  # First 10 tags
                section.append("")
            
            # Thumbnail
            thumbnail = video.get('thumbnail', '')
            if thumbnail:
                section.append(f"## Thumbnail: {thumbnail}")
                section.append("")
            
            # Full Description (untruncated!)
            section.append("## Description:")
            description = video.get('description', '')
            if description:
                section.append(description)
            section.append("")
            
            # Caption/Transcript
            caption_text = video.get('caption_text')
            if caption_text:
                section.append("## Caption Status:")
                section.append(caption_text)
                section.append("")
            
            # Separator
            section.append("-" * 50)
            section.append("")
            
            markdown_sections.append('\n'.join(section))
        
        return '\n'.join(markdown_sections)
    
    def get_incremental_items(self, items: List[Dict[str, Any]], state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Get only new videos since last sync."""
        if not state:
            return items
        
        last_video_id = state.get('last_video_id')
        last_published = state.get('last_published')
        
        if not last_video_id:
            return items
        
        # Filter for videos newer than the last synced
        new_items = []
        for item in items:
            if item.get('id') == last_video_id:
                break  # Found the last synced video
            
            # Also check by publish date as backup
            if last_published and item.get('published_at'):
                if item['published_at'] <= last_published:
                    continue
            
            new_items.append(item)
        
        return new_items
    
    def update_state(self, state: Dict[str, Any], items: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Update state with latest video information."""
        if not items:
            return state
        
        # Get the first item (most recent)
        latest_item = items[0]
        
        state['last_video_id'] = latest_item.get('id')
        state['last_published'] = latest_item.get('published_at')
        state['last_video_title'] = latest_item.get('title')
        state['last_sync'] = datetime.now(self.tz).isoformat()
        state['video_count'] = len(items)
        state['quota_used'] = self.quota_used
        
        return state