#!/usr/bin/env python3
"""
Cumulative Markdown Manager
Maintains a single, growing markdown file per source that combines:
- Initial backlog content
- Daily incremental updates
- Updates to existing entries (e.g., new captions, updated metrics)
"""

from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional, Any
import pytz
import logging
import shutil
import re


class CumulativeMarkdownManager:
    """Manages cumulative markdown files that grow with each update."""
    
    def __init__(self, config, logger: Optional[logging.Logger] = None):
        """Initialize with scraper config."""
        self.config = config
        self.logger = logger or logging.getLogger(self.__class__.__name__)
        self.tz = pytz.timezone(config.timezone)
        
        # Paths
        self.current_dir = config.data_dir / "markdown_current"
        self.archive_dir = config.data_dir / "markdown_archives" / config.source_name.title()
        
        # Ensure directories exist
        self.current_dir.mkdir(parents=True, exist_ok=True)
        self.archive_dir.mkdir(parents=True, exist_ok=True)
        
        # File pattern for this source
        self.file_pattern = f"{config.brand_name}_{config.source_name}_*.md"
        
    def get_current_file(self) -> Optional[Path]:
        """Find the current markdown file for this source."""
        files = list(self.current_dir.glob(self.file_pattern))
        if not files:
            return None
        
        # Return the most recent file (by filename timestamp)
        files.sort(reverse=True)
        return files[0]
    
    def parse_markdown_sections(self, content: str) -> Dict[str, Dict]:
        """Parse markdown content into sections indexed by ID."""
        sections = {}
        
        # Split by ID headers
        parts = content.split('# ID: ')
        
        for part in parts[1:]:  # Skip first empty part
            if not part.strip():
                continue
                
            lines = part.strip().split('\n')
            section_id = lines[0].strip()
            
            # Reconstruct full section content
            section_content = f"# ID: {section_id}\n" + '\n'.join(lines[1:])
            
            # Extract metadata for comparison
            metadata = self._extract_metadata(section_content)
            
            sections[section_id] = {
                'id': section_id,
                'content': section_content,
                'metadata': metadata
            }
        
        return sections
    
    def _extract_metadata(self, content: str) -> Dict[str, Any]:
        """Extract metadata from section content for comparison."""
        metadata = {}
        
        # Extract common fields
        patterns = {
            'views': r'## Views?:\s*([0-9,]+)',
            'likes': r'## Likes?:\s*([0-9,]+)',
            'comments': r'## Comments?:\s*([0-9,]+)',
            'publish_date': r'## Publish(?:ed)? Date:\s*([^\n]+)',
            'has_caption': r'## Caption Status:',
            'has_transcript': r'## Transcript:',
            'description_length': r'## Description:\n(.+?)(?:\n##|\n---|\Z)',
        }
        
        for key, pattern in patterns.items():
            match = re.search(pattern, content, re.DOTALL | re.IGNORECASE)
            if match:
                if key in ['views', 'likes', 'comments']:
                    # Convert numeric fields
                    metadata[key] = int(match.group(1).replace(',', ''))
                elif key in ['has_caption', 'has_transcript']:
                    # Boolean fields
                    metadata[key] = True
                elif key == 'description_length':
                    # Calculate length of description
                    metadata[key] = len(match.group(1).strip())
                else:
                    metadata[key] = match.group(1).strip()
        
        return metadata
    
    def should_update_section(self, old_section: Dict, new_section: Dict) -> bool:
        """Determine if a section should be updated with new content."""
        old_meta = old_section.get('metadata', {})
        new_meta = new_section.get('metadata', {})
        
        # Update if new section has captions/transcripts that old doesn't
        if new_meta.get('has_caption') and not old_meta.get('has_caption'):
            return True
        if new_meta.get('has_transcript') and not old_meta.get('has_transcript'):
            return True
        
        # Update if new section has more content
        old_desc_len = old_meta.get('description_length', 0)
        new_desc_len = new_meta.get('description_length', 0)
        if new_desc_len > old_desc_len * 1.2:  # 20% more content
            return True
        
        # Update if metrics have changed significantly (for incremental updates)
        for metric in ['views', 'likes', 'comments']:
            old_val = old_meta.get(metric, 0)
            new_val = new_meta.get(metric, 0)
            if new_val > old_val:
                return True
        
        # Update if content is substantially different
        if len(new_section['content']) > len(old_section['content']) * 1.1:
            return True
            
        return False
    
    def merge_content(self, existing_sections: Dict[str, Dict], 
                     new_items: List[Dict[str, Any]], 
                     formatter_func) -> str:
        """Merge new content with existing sections."""
        # Convert new items to sections
        new_content = formatter_func(new_items)
        new_sections = self.parse_markdown_sections(new_content)
        
        # Track updates
        added_count = 0
        updated_count = 0
        
        # Merge sections
        for section_id, new_section in new_sections.items():
            if section_id in existing_sections:
                # Update existing section if newer/better
                if self.should_update_section(existing_sections[section_id], new_section):
                    existing_sections[section_id] = new_section
                    updated_count += 1
                    self.logger.info(f"Updated section: {section_id}")
            else:
                # Add new section
                existing_sections[section_id] = new_section
                added_count += 1
                self.logger.debug(f"Added new section: {section_id}")
        
        self.logger.info(f"Merge complete: {added_count} added, {updated_count} updated")
        
        # Reconstruct markdown content
        # Sort by ID to maintain consistent order
        sorted_sections = sorted(existing_sections.values(), 
                               key=lambda x: x['id'])
        
        # For sources with dates, sort by date (newest first)
        # Try to extract date from content for better sorting
        for section in sorted_sections:
            date_match = re.search(r'## Publish(?:ed)? Date:\s*([^\n]+)', 
                                  section['content'])
            if date_match:
                try:
                    # Parse various date formats
                    date_str = date_match.group(1).strip()
                    # Add parsed date for sorting
                    section['sort_date'] = date_str
                except:
                    pass
        
        # Sort by date if available, otherwise by ID
        if any('sort_date' in s for s in sorted_sections):
            sorted_sections.sort(key=lambda x: x.get('sort_date', ''), reverse=True)
        
        # Combine sections
        combined_content = []
        for section in sorted_sections:
            combined_content.append(section['content'])
            combined_content.append("")  # Empty line between sections
        
        return '\n'.join(combined_content)
    
    def save_cumulative(self, new_items: List[Dict[str, Any]], 
                       formatter_func) -> Path:
        """Save content cumulatively, merging with existing file if present."""
        current_file = self.get_current_file()
        
        if current_file and current_file.exists():
            # Load and merge with existing content
            self.logger.info(f"Loading existing file: {current_file.name}")
            existing_content = current_file.read_text(encoding='utf-8')
            existing_sections = self.parse_markdown_sections(existing_content)
            
            # Merge new items with existing sections
            merged_content = self.merge_content(existing_sections, new_items, 
                                              formatter_func)
            
            # Archive the current file before overwriting
            self._archive_file(current_file)
            
        else:
            # First time - just format the new items
            self.logger.info("No existing file, creating new cumulative file")
            merged_content = formatter_func(new_items)
        
        # Generate new filename with current timestamp
        timestamp = datetime.now(self.tz).strftime('%Y-%m-%dT%H%M%S')
        filename = f"{self.config.brand_name}_{self.config.source_name}_{timestamp}.md"
        filepath = self.current_dir / filename
        
        # Save merged content
        filepath.write_text(merged_content, encoding='utf-8')
        self.logger.info(f"Saved cumulative file: {filename}")
        
        # Remove old file if it exists (we archived it already)
        if current_file and current_file.exists() and current_file != filepath:
            current_file.unlink()
            self.logger.debug(f"Removed old file: {current_file.name}")
        
        return filepath
    
    def _archive_file(self, file_path: Path) -> None:
        """Archive a file with timestamp suffix."""
        if not file_path.exists():
            return
        
        # Add archive timestamp to filename
        archive_time = datetime.now(self.tz).strftime('%Y%m%d_%H%M%S')
        archive_name = f"{file_path.stem}_archived_{archive_time}{file_path.suffix}"
        archive_path = self.archive_dir / archive_name
        
        # Copy to archive
        shutil.copy2(file_path, archive_path)
        self.logger.debug(f"Archived to: {archive_path.name}")
    
    def get_statistics(self, file_path: Optional[Path] = None) -> Dict[str, int]:
        """Get statistics about the cumulative file."""
        if not file_path:
            file_path = self.get_current_file()
        
        if not file_path or not file_path.exists():
            return {'total_sections': 0}
        
        content = file_path.read_text(encoding='utf-8')
        sections = self.parse_markdown_sections(content)
        
        stats = {
            'total_sections': len(sections),
            'with_captions': sum(1 for s in sections.values() 
                               if s['metadata'].get('has_caption')),
            'with_transcripts': sum(1 for s in sections.values() 
                                  if s['metadata'].get('has_transcript')),
            'total_views': sum(s['metadata'].get('views', 0) 
                             for s in sections.values()),
            'file_size_kb': file_path.stat().st_size // 1024
        }
        
        return stats
    
    def update_cumulative_file(self, items: List[Dict[str, Any]], source_name: str) -> Path:
        """
        Update cumulative file for a source using a basic formatter.
        This is a compatibility method for scripts that expect this interface.
        """
        def basic_formatter(items: List[Dict[str, Any]]) -> str:
            """Basic markdown formatter for any source."""
            sections = []
            
            for item in items:
                section = []
                
                # ID
                item_id = item.get('id', 'Unknown')
                section.append(f"# ID: {item_id}")
                section.append("")
                
                # Title
                title = item.get('title', item.get('caption', 'Untitled'))
                if title:
                    # Truncate very long titles/captions
                    if len(title) > 100:
                        title = title[:97] + "..."
                    section.append(f"## Title: {title}")
                    section.append("")
                
                # Type
                item_type = item.get('type', source_name.lower())
                section.append(f"## Type: {item_type}")
                section.append("")
                
                # Link
                link = item.get('link', item.get('url', ''))
                if link:
                    section.append(f"## Link: {link}")
                    section.append("")
                
                # Author/Channel
                author = item.get('author', item.get('channel', ''))
                if author:
                    section.append(f"## Author: {author}")
                    section.append("")
                
                # Publish Date
                pub_date = item.get('publish_date', item.get('published', ''))
                if pub_date:
                    section.append(f"## Publish Date: {pub_date}")
                    section.append("")
                
                # Views
                views = item.get('views')
                if views is not None:
                    section.append(f"## Views: {views:,}")
                    section.append("")
                
                # Likes
                likes = item.get('likes')
                if likes is not None:
                    section.append(f"## Likes: {likes:,}")
                    section.append("")
                
                # Comments
                comments = item.get('comments')
                if comments is not None:
                    section.append(f"## Comments: {comments:,}")
                    section.append("")
                
                # Local images
                local_images = item.get('local_images', [])
                if local_images:
                    section.append(f"## Images Downloaded: {len(local_images)}")
                    for i, img_path in enumerate(local_images, 1):
                        rel_path = Path(img_path).relative_to(self.config.data_dir)
                        section.append(f"![Image {i}]({rel_path})")
                    section.append("")
                
                # Local thumbnail
                local_thumbnail = item.get('local_thumbnail')
                if local_thumbnail:
                    section.append("## Thumbnail:")
                    rel_path = Path(local_thumbnail).relative_to(self.config.data_dir)
                    section.append(f"![Thumbnail]({rel_path})")
                    section.append("")
                
                # Description/Caption
                description = item.get('description', item.get('caption', ''))
                if description:
                    section.append("## Description:")
                    section.append(description)
                    section.append("")
                
                # Separator
                section.append("-" * 50)
                section.append("")
                
                sections.append('\n'.join(section))
            
            return '\n'.join(sections)
        
        return self.save_cumulative(items, basic_formatter)