"""
HVAC Content Backlog Processor

Processes all existing markdown files to create classified JSON summaries.
"""

import asyncio
import json
from pathlib import Path
from typing import List, Dict, Any, Optional
import logging
from datetime import datetime
import argparse

from .content_parser import ContentParser, ContentItem
from .content_classifier import ContentClassifier, ClassifiedContent

logger = logging.getLogger(__name__)


class BacklogProcessor:
    """Processes backlog of HVAC content files."""
    
    def __init__(self, data_dir: Path, output_dir: Path, api_key: Optional[str] = None):
        self.data_dir = Path(data_dir)
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
        
        self.parser = ContentParser()
        self.classifier = ContentClassifier(api_key=api_key)
        
        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
    
    def discover_markdown_files(self) -> Dict[str, List[Path]]:
        """Discover all markdown files for processing."""
        file_groups = {
            "current": [],
            "archives": []
        }
        
        # Current files
        current_dir = self.data_dir / "markdown_current"
        if current_dir.exists():
            current_files = list(current_dir.glob("hkia_*.md"))
            file_groups["current"] = current_files
            self.logger.info(f"Found {len(current_files)} current markdown files")
        
        # Archive files
        archive_dir = self.data_dir / "markdown_archives"
        if archive_dir.exists():
            archive_files = []
            for source_dir in archive_dir.iterdir():
                if source_dir.is_dir():
                    source_files = list(source_dir.glob("hkia_*.md"))
                    archive_files.extend(source_files)
            
            file_groups["archives"] = archive_files
            self.logger.info(f"Found {len(archive_files)} archived markdown files")
        
        total_files = len(file_groups["current"]) + len(file_groups["archives"])
        self.logger.info(f"Total markdown files discovered: {total_files}")
        
        return file_groups
    
    async def process_file_group(self, files: List[Path], group_name: str, 
                                batch_size: int = 20) -> List[ClassifiedContent]:
        """Process a group of files (current or archives)."""
        self.logger.info(f"Processing {group_name} files: {len(files)} files")
        
        all_classified = []
        
        for file_path in files:
            try:
                self.logger.info(f"Processing file: {file_path.name}")
                
                # Parse content items from file
                items = self.parser.parse_markdown_file(file_path)
                self.logger.info(f"Extracted {len(items)} content items from {file_path.name}")
                
                if items:
                    # Classify content items in batches
                    classified_items = await self.classifier.classify_content_batch(
                        items, batch_size=batch_size
                    )
                    
                    all_classified.extend(classified_items)
                    
                    # Save individual file results
                    await self._save_file_results(file_path, classified_items)
                    
                    self.logger.info(f"Completed {file_path.name}: "
                                   f"{len(classified_items)}/{len(items)} items classified")
                
            except Exception as e:
                self.logger.error(f"Error processing file {file_path}: {e}")
        
        self.logger.info(f"Completed {group_name} processing: {len(all_classified)} total items classified")
        return all_classified
    
    async def _save_file_results(self, source_file: Path, classified_items: List[ClassifiedContent]):
        """Save classification results for individual files."""
        if not classified_items:
            return
            
        # Create output filename based on source file
        base_name = source_file.stem  # Remove .md extension
        output_file = self.output_dir / f"{base_name}_classified.json"
        
        try:
            data = {
                "source_file": str(source_file),
                "processed_at": datetime.now().isoformat(),
                "total_items": len(classified_items),
                "classified_content": [item.__dict__ for item in classified_items]
            }
            
            with open(output_file, 'w', encoding='utf-8') as f:
                json.dump(data, f, indent=2, ensure_ascii=False)
            
            self.logger.debug(f"Saved individual results to {output_file}")
            
        except Exception as e:
            self.logger.error(f"Error saving individual file results: {e}")
    
    async def process_backlog(self, batch_size: int = 20, 
                            process_current: bool = True,
                            process_archives: bool = True) -> Dict[str, Any]:
        """Process entire backlog of content files."""
        self.logger.info("Starting backlog processing...")
        start_time = datetime.now()
        
        # Discover files
        file_groups = self.discover_markdown_files()
        
        results = {
            "started_at": start_time.isoformat(),
            "files_discovered": {
                "current": len(file_groups["current"]),
                "archives": len(file_groups["archives"])
            },
            "classified_content": {
                "current": [],
                "archives": []
            },
            "summary": {}
        }
        
        # Process current files
        if process_current and file_groups["current"]:
            current_classified = await self.process_file_group(
                file_groups["current"], "current", batch_size
            )
            results["classified_content"]["current"] = current_classified
        
        # Process archive files
        if process_archives and file_groups["archives"]:
            archive_classified = await self.process_file_group(
                file_groups["archives"], "archives", batch_size
            )
            results["classified_content"]["archives"] = archive_classified
        
        # Generate summary
        end_time = datetime.now()
        total_classified = (len(results["classified_content"]["current"]) + 
                          len(results["classified_content"]["archives"]))
        
        results["completed_at"] = end_time.isoformat()
        results["duration_seconds"] = (end_time - start_time).total_seconds()
        results["summary"] = {
            "total_items_classified": total_classified,
            "current_items": len(results["classified_content"]["current"]),
            "archive_items": len(results["classified_content"]["archives"]),
            "processing_rate_items_per_minute": round(
                total_classified / (results["duration_seconds"] / 60), 2
            ) if results["duration_seconds"] > 0 else 0
        }
        
        # Save comprehensive results
        await self._save_comprehensive_results(results)
        
        self.logger.info(f"Backlog processing complete!")
        self.logger.info(f"Total items classified: {total_classified}")
        self.logger.info(f"Processing time: {results['duration_seconds']:.1f} seconds")
        self.logger.info(f"Rate: {results['summary']['processing_rate_items_per_minute']} items/min")
        
        return results
    
    async def _save_comprehensive_results(self, results: Dict[str, Any]):
        """Save comprehensive backlog processing results."""
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        
        # Save full results
        full_results_file = self.output_dir / f"backlog_classification_complete_{timestamp}.json"
        
        # Convert ClassifiedContent objects to dicts for JSON serialization
        serializable_results = results.copy()
        for group in ["current", "archives"]:
            serializable_results["classified_content"][group] = [
                item.__dict__ if hasattr(item, '__dict__') else item 
                for item in results["classified_content"][group]
            ]
        
        with open(full_results_file, 'w', encoding='utf-8') as f:
            json.dump(serializable_results, f, indent=2, ensure_ascii=False)
        
        # Save summary only
        summary_file = self.output_dir / f"backlog_classification_summary_{timestamp}.json"
        summary_data = {
            "processing_summary": results["summary"],
            "started_at": results["started_at"], 
            "completed_at": results["completed_at"],
            "files_processed": results["files_discovered"]
        }
        
        with open(summary_file, 'w', encoding='utf-8') as f:
            json.dump(summary_data, f, indent=2, ensure_ascii=False)
        
        self.logger.info(f"Saved comprehensive results to {full_results_file}")
        self.logger.info(f"Saved summary to {summary_file}")
    
    def generate_statistics_report(self, classified_items: List[ClassifiedContent]) -> Dict[str, Any]:
        """Generate statistics report from classified content."""
        if not classified_items:
            return {}
        
        # Count by source
        source_counts = {}
        content_type_counts = {}
        application_counts = {}
        brand_counts = {}
        tool_counts = {}
        topic_counts = {}
        
        for item in classified_items:
            # Source distribution
            source_counts[item.source] = source_counts.get(item.source, 0) + 1
            
            # Content type distribution
            content_type_counts[item.content_type] = content_type_counts.get(item.content_type, 0) + 1
            
            # Application distribution
            for app in item.application:
                application_counts[app] = application_counts.get(app, 0) + 1
            
            # Brand mentions
            for brand in item.brands_mentioned:
                brand_counts[brand] = brand_counts.get(brand, 0) + 1
                
            # Tool mentions
            for tool in item.tools_mentioned:
                tool_counts[tool] = tool_counts.get(tool, 0) + 1
                
            # Topic mentions
            for topic in item.topics:
                topic_counts[topic] = topic_counts.get(topic, 0) + 1
        
        return {
            "total_items": len(classified_items),
            "source_distribution": dict(sorted(source_counts.items(), key=lambda x: x[1], reverse=True)),
            "content_type_distribution": dict(sorted(content_type_counts.items(), key=lambda x: x[1], reverse=True)),
            "application_distribution": dict(sorted(application_counts.items(), key=lambda x: x[1], reverse=True)),
            "top_brands_mentioned": dict(sorted(brand_counts.items(), key=lambda x: x[1], reverse=True)[:20]),
            "top_tools_mentioned": dict(sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)[:20]),
            "top_topics": dict(sorted(topic_counts.items(), key=lambda x: x[1], reverse=True)[:30])
        }


async def main():
    """Main entry point for backlog processing."""
    parser = argparse.ArgumentParser(description="Process HVAC content backlog")
    parser.add_argument("--data-dir", type=Path, default=Path("data"),
                       help="Data directory containing markdown files")
    parser.add_argument("--output-dir", type=Path, default=Path("data/classified_content"),
                       help="Output directory for classified content")
    parser.add_argument("--batch-size", type=int, default=20,
                       help="Batch size for classification")
    parser.add_argument("--current-only", action="store_true",
                       help="Process only current files, skip archives")
    parser.add_argument("--archives-only", action="store_true", 
                       help="Process only archive files, skip current")
    
    args = parser.parse_args()
    
    # Set up logging
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )
    
    # Initialize processor
    processor = BacklogProcessor(
        data_dir=args.data_dir,
        output_dir=args.output_dir
    )
    
    # Process backlog
    process_current = not args.archives_only
    process_archives = not args.current_only
    
    results = await processor.process_backlog(
        batch_size=args.batch_size,
        process_current=process_current,
        process_archives=process_archives
    )
    
    # Generate statistics
    all_classified = (results["classified_content"]["current"] + 
                     results["classified_content"]["archives"])
    
    if all_classified:
        stats = processor.generate_statistics_report(all_classified)
        
        print("\n" + "="*60)
        print("BACKLOG PROCESSING STATISTICS")
        print("="*60)
        print(f"Total items classified: {stats['total_items']}")
        print(f"\nTop sources:")
        for source, count in list(stats['source_distribution'].items())[:5]:
            print(f"  {source}: {count}")
            
        print(f"\nTop content types:")
        for ctype, count in list(stats['content_type_distribution'].items())[:5]:
            print(f"  {ctype}: {count}")
            
        print(f"\nTop topics:")
        for topic, count in list(stats['top_topics'].items())[:10]:
            print(f"  {topic}: {count}")


if __name__ == "__main__":
    asyncio.run(main())