import os
import json
import time
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
import requests
import pytz
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type

from src.base_scraper import BaseScraper, ScraperConfig


@dataclass
class CompetitiveConfig:
    """Extended configuration for competitive intelligence scrapers."""
    source_name: str
    brand_name: str
    data_dir: Path
    logs_dir: Path
    competitor_name: str
    base_url: str
    timezone: str = "America/Halifax"
    use_proxy: bool = True
    proxy_rotation: bool = True
    max_concurrent_requests: int = 2
    request_delay: float = 3.0
    backlog_limit: int = 100  # For initial backlog capture


class BaseCompetitiveScraper(BaseScraper):
    """Base class for competitive intelligence scrapers with proxy support and advanced anti-detection."""
    
    def __init__(self, config: CompetitiveConfig):
        # Create a ScraperConfig for the parent class
        scraper_config = ScraperConfig(
            source_name=config.source_name,
            brand_name=config.brand_name,
            data_dir=config.data_dir,
            logs_dir=config.logs_dir,
            timezone=config.timezone
        )
        super().__init__(scraper_config)
        self.competitive_config = config
        self.competitor_name = config.competitor_name
        self.base_url = config.base_url
        
        # Proxy configuration from environment
        self.oxylabs_config = {
            'username': os.getenv('OXYLABS_USERNAME'),
            'password': os.getenv('OXYLABS_PASSWORD'),
            'endpoint': os.getenv('OXYLABS_PROXY_ENDPOINT', 'pr.oxylabs.io'),
            'port': int(os.getenv('OXYLABS_PROXY_PORT', '7777'))
        }
        
        # Jina.ai configuration for content extraction
        self.jina_api_key = os.getenv('JINA_API_KEY')
        
        # Enhanced rate limiting for competitive scraping
        self.request_delay = config.request_delay
        self.last_request_time = 0
        self.max_concurrent_requests = config.max_concurrent_requests
        
        # Setup competitive intelligence specific directories
        self._setup_competitive_directories()
        
        # Configure session with proxy if enabled
        if config.use_proxy and self.oxylabs_config['username']:
            self._configure_proxy_session()
        
        # Enhanced user agent pool for competitive scraping
        self.competitive_user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/120.0.0.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
        ]
        
        # Content cache to avoid re-scraping
        self.content_cache = {}
        
        # Initialize state management for competitive intelligence
        self.competitive_state_file = config.data_dir / ".state" / f"competitive_{config.competitor_name}_state.json"
        
        self.logger.info(f"Initialized competitive scraper for {self.competitor_name}")

    def _setup_competitive_directories(self):
        """Create directories specific to competitive intelligence."""
        # Create competitive intelligence specific directories
        comp_dir = self.config.data_dir / "competitive_intelligence" / self.competitor_name
        comp_dir.mkdir(parents=True, exist_ok=True)
        
        # Subdirectories for different types of content
        (comp_dir / "backlog").mkdir(exist_ok=True)
        (comp_dir / "incremental").mkdir(exist_ok=True)
        (comp_dir / "analysis").mkdir(exist_ok=True)
        (comp_dir / "media").mkdir(exist_ok=True)
        
        # State directory for competitive intelligence
        state_dir = self.config.data_dir / ".state" / "competitive"
        state_dir.mkdir(parents=True, exist_ok=True)

    def _configure_proxy_session(self):
        """Configure HTTP session with Oxylabs proxy."""
        try:
            proxy_url = f"http://{self.oxylabs_config['username']}:{self.oxylabs_config['password']}@{self.oxylabs_config['endpoint']}:{self.oxylabs_config['port']}"
            
            proxies = {
                'http': proxy_url,
                'https': proxy_url
            }
            
            self.session.proxies.update(proxies)
            
            # Test proxy connection
            test_response = self.session.get('http://httpbin.org/ip', timeout=10)
            if test_response.status_code == 200:
                proxy_ip = test_response.json().get('origin', 'Unknown')
                self.logger.info(f"Proxy connection established. IP: {proxy_ip}")
            else:
                self.logger.warning("Proxy test failed, continuing with direct connection")
                self.session.proxies.clear()
                
        except Exception as e:
            self.logger.warning(f"Failed to configure proxy: {e}. Using direct connection.")
            self.session.proxies.clear()

    def _apply_competitive_rate_limit(self):
        """Apply enhanced rate limiting for competitive scraping."""
        current_time = time.time()
        time_since_last = current_time - self.last_request_time
        
        if time_since_last < self.request_delay:
            sleep_time = self.request_delay - time_since_last
            self.logger.debug(f"Rate limiting: sleeping for {sleep_time:.2f} seconds")
            time.sleep(sleep_time)
        
        self.last_request_time = time.time()

    def rotate_competitive_user_agent(self):
        """Rotate user agent from competitive pool."""
        import random
        user_agent = random.choice(self.competitive_user_agents)
        self.session.headers.update({'User-Agent': user_agent})
        self.logger.debug(f"Rotated to competitive user agent: {user_agent[:50]}...")

    def make_competitive_request(self, url: str, **kwargs) -> requests.Response:
        """Make HTTP request with competitive intelligence optimizations."""
        self._apply_competitive_rate_limit()
        
        # Rotate user agent for each request
        self.rotate_competitive_user_agent()
        
        # Add additional headers to appear more browser-like
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
        }
        
        # Merge with existing headers
        if 'headers' in kwargs:
            headers.update(kwargs['headers'])
        kwargs['headers'] = headers
        
        # Set timeout if not specified
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 30
        
        @self.get_retry_decorator()
        def _make_request():
            return self.session.get(url, **kwargs)
        
        return _make_request()

    def extract_with_jina(self, url: str) -> Optional[Dict[str, Any]]:
        """Extract content using Jina.ai Reader API."""
        if not self.jina_api_key:
            self.logger.warning("Jina API key not configured, skipping AI extraction")
            return None
        
        try:
            jina_url = f"https://r.jina.ai/{url}"
            headers = {
                'Authorization': f'Bearer {self.jina_api_key}',
                'X-With-Generated-Alt': 'true'
            }
            
            response = requests.get(jina_url, headers=headers, timeout=30)
            response.raise_for_status()
            
            content = response.text
            
            # Parse response (Jina returns markdown format)
            return {
                'content': content,
                'extraction_method': 'jina_ai',
                'extraction_timestamp': datetime.now(self.tz).isoformat()
            }
            
        except Exception as e:
            self.logger.error(f"Jina extraction failed for {url}: {e}")
            return None

    def load_competitive_state(self) -> Dict[str, Any]:
        """Load competitive intelligence specific state."""
        if not self.competitive_state_file.exists():
            self.logger.info(f"No competitive state file found for {self.competitor_name}, starting fresh")
            return {
                'last_backlog_capture': None,
                'last_incremental_sync': None,
                'total_items_captured': 0,
                'content_urls': set(),
                'competitor_name': self.competitor_name,
                'initialized': datetime.now(self.tz).isoformat()
            }
        
        try:
            with open(self.competitive_state_file, 'r') as f:
                state = json.load(f)
                # Convert content_urls back to set
                if 'content_urls' in state and isinstance(state['content_urls'], list):
                    state['content_urls'] = set(state['content_urls'])
                return state
        except Exception as e:
            self.logger.error(f"Error loading competitive state: {e}")
            return {}

    def save_competitive_state(self, state: Dict[str, Any]) -> None:
        """Save competitive intelligence specific state."""
        try:
            # Convert set to list for JSON serialization
            state_copy = state.copy()
            if 'content_urls' in state_copy and isinstance(state_copy['content_urls'], set):
                state_copy['content_urls'] = list(state_copy['content_urls'])
            
            self.competitive_state_file.parent.mkdir(parents=True, exist_ok=True)
            with open(self.competitive_state_file, 'w') as f:
                json.dump(state_copy, f, indent=2)
            self.logger.debug(f"Saved competitive state for {self.competitor_name}")
        except Exception as e:
            self.logger.error(f"Error saving competitive state: {e}")

    def generate_competitive_filename(self, content_type: str = "incremental") -> str:
        """Generate filename for competitive intelligence content."""
        now = datetime.now(self.tz)
        timestamp = now.strftime("%Y%m%d_%H%M%S")
        return f"competitive_{self.competitor_name}_{content_type}_{timestamp}.md"

    def save_competitive_content(self, content: str, content_type: str = "incremental") -> Path:
        """Save content to competitive intelligence directories."""
        filename = self.generate_competitive_filename(content_type)
        
        # Determine output directory based on content type
        if content_type == "backlog":
            output_dir = self.config.data_dir / "competitive_intelligence" / self.competitor_name / "backlog"
        elif content_type == "analysis":
            output_dir = self.config.data_dir / "competitive_intelligence" / self.competitor_name / "analysis"
        else:
            output_dir = self.config.data_dir / "competitive_intelligence" / self.competitor_name / "incremental"
        
        output_dir.mkdir(parents=True, exist_ok=True)
        filepath = output_dir / filename
        
        try:
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(content)
            self.logger.info(f"Saved {content_type} content to {filepath}")
            return filepath
        except Exception as e:
            self.logger.error(f"Error saving {content_type} content: {e}")
            raise

    @abstractmethod
    def discover_content_urls(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
        """Discover content URLs from competitor site (sitemap, RSS, pagination, etc.)."""
        pass

    @abstractmethod
    def scrape_content_item(self, url: str) -> Optional[Dict[str, Any]]:
        """Scrape individual content item from competitor."""
        pass

    def run_backlog_capture(self, limit: Optional[int] = None) -> None:
        """Run initial backlog capture for competitor content."""
        try:
            self.logger.info(f"Starting backlog capture for {self.competitor_name} (limit: {limit})")
            
            # Load state
            state = self.load_competitive_state()
            
            # Discover content URLs
            content_urls = self.discover_content_urls(limit or self.competitive_config.backlog_limit)
            
            if not content_urls:
                self.logger.warning("No content URLs discovered")
                return
            
            self.logger.info(f"Discovered {len(content_urls)} content URLs")
            
            # Scrape content items
            scraped_items = []
            for i, url_data in enumerate(content_urls, 1):
                url = url_data.get('url') if isinstance(url_data, dict) else url_data
                self.logger.info(f"Scraping item {i}/{len(content_urls)}: {url}")
                
                item = self.scrape_content_item(url)
                if item:
                    scraped_items.append(item)
                
                # Progress logging
                if i % 10 == 0:
                    self.logger.info(f"Completed {i}/{len(content_urls)} items")
            
            if scraped_items:
                # Format as markdown
                markdown_content = self.format_competitive_markdown(scraped_items)
                
                # Save backlog content
                filepath = self.save_competitive_content(markdown_content, "backlog")
                
                # Update state
                state['last_backlog_capture'] = datetime.now(self.tz).isoformat()
                state['total_items_captured'] = len(scraped_items)
                if 'content_urls' not in state:
                    state['content_urls'] = set()
                
                for item in scraped_items:
                    if 'url' in item:
                        state['content_urls'].add(item['url'])
                
                self.save_competitive_state(state)
                
                self.logger.info(f"Backlog capture complete: {len(scraped_items)} items saved to {filepath}")
            else:
                self.logger.warning("No items successfully scraped during backlog capture")
                
        except Exception as e:
            self.logger.error(f"Error in backlog capture: {e}")
            raise

    def run_incremental_sync(self) -> None:
        """Run incremental sync for new competitor content."""
        try:
            self.logger.info(f"Starting incremental sync for {self.competitor_name}")
            
            # Load state
            state = self.load_competitive_state()
            known_urls = state.get('content_urls', set())
            
            # Discover new content URLs
            all_content_urls = self.discover_content_urls(50)  # Check recent items
            
            # Filter for new URLs only
            new_urls = []
            for url_data in all_content_urls:
                url = url_data.get('url') if isinstance(url_data, dict) else url_data
                if url not in known_urls:
                    new_urls.append(url_data)
            
            if not new_urls:
                self.logger.info("No new content found during incremental sync")
                return
            
            self.logger.info(f"Found {len(new_urls)} new content items")
            
            # Scrape new content items
            new_items = []
            for url_data in new_urls:
                url = url_data.get('url') if isinstance(url_data, dict) else url_data
                self.logger.debug(f"Scraping new item: {url}")
                
                item = self.scrape_content_item(url)
                if item:
                    new_items.append(item)
            
            if new_items:
                # Format as markdown
                markdown_content = self.format_competitive_markdown(new_items)
                
                # Save incremental content
                filepath = self.save_competitive_content(markdown_content, "incremental")
                
                # Update state
                state['last_incremental_sync'] = datetime.now(self.tz).isoformat()
                state['total_items_captured'] = state.get('total_items_captured', 0) + len(new_items)
                
                for item in new_items:
                    if 'url' in item:
                        state['content_urls'].add(item['url'])
                
                self.save_competitive_state(state)
                
                self.logger.info(f"Incremental sync complete: {len(new_items)} new items saved to {filepath}")
            else:
                self.logger.info("No new items successfully scraped during incremental sync")
                
        except Exception as e:
            self.logger.error(f"Error in incremental sync: {e}")
            raise

    def format_competitive_markdown(self, items: List[Dict[str, Any]]) -> str:
        """Format competitive intelligence items as markdown."""
        if not items:
            return ""
        
        # Add header with competitive intelligence metadata
        header_lines = [
            f"# Competitive Intelligence: {self.competitor_name}",
            f"",
            f"**Source**: {self.base_url}",
            f"**Capture Date**: {datetime.now(self.tz).strftime('%Y-%m-%d %H:%M:%S %Z')}",
            f"**Items Captured**: {len(items)}",
            f"",
            f"---",
            f""
        ]
        
        # Format each item
        formatted_items = []
        for item in items:
            formatted_item = self.format_competitive_item(item)
            formatted_items.append(formatted_item)
        
        # Combine header and items
        content = "\n".join(header_lines) + "\n\n".join(formatted_items)
        
        return content

    def format_competitive_item(self, item: Dict[str, Any]) -> str:
        """Format a single competitive intelligence item."""
        lines = []
        
        # ID
        item_id = item.get('id', item.get('url', 'unknown'))
        lines.append(f"# ID: {item_id}")
        lines.append("")
        
        # Title
        title = item.get('title', 'Untitled')
        lines.append(f"## Title: {title}")
        lines.append("")
        
        # Competitor
        lines.append(f"## Competitor: {self.competitor_name}")
        lines.append("")
        
        # Type
        content_type = item.get('type', 'unknown')
        lines.append(f"## Type: {content_type}")
        lines.append("")
        
        # Permalink
        permalink = item.get('url', 'N/A')
        lines.append(f"## Permalink: {permalink}")
        lines.append("")
        
        # Publish Date
        publish_date = item.get('publish_date', item.get('date', 'Unknown'))
        lines.append(f"## Publish Date: {publish_date}")
        lines.append("")
        
        # Author
        author = item.get('author', 'Unknown')
        lines.append(f"## Author: {author}")
        lines.append("")
        
        # Word Count
        word_count = item.get('word_count', 'Unknown')
        lines.append(f"## Word Count: {word_count}")
        lines.append("")
        
        # Categories/Tags
        categories = item.get('categories', item.get('tags', []))
        if categories:
            if isinstance(categories, list):
                categories_str = ', '.join(categories)
            else:
                categories_str = str(categories)
        else:
            categories_str = 'None'
        lines.append(f"## Categories: {categories_str}")
        lines.append("")
        
        # Competitive Intelligence Metadata
        lines.append("## Intelligence Metadata:")
        lines.append("")
        
        # Scraping method
        extraction_method = item.get('extraction_method', 'standard_scraping')
        lines.append(f"### Extraction Method: {extraction_method}")
        lines.append("")
        
        # Capture timestamp
        capture_time = item.get('capture_timestamp', datetime.now(self.tz).isoformat())
        lines.append(f"### Captured: {capture_time}")
        lines.append("")
        
        # Social metrics (if available)
        if 'social_metrics' in item:
            metrics = item['social_metrics']
            lines.append("### Social Metrics:")
            for metric, value in metrics.items():
                lines.append(f"- {metric.title()}: {value}")
            lines.append("")
        
        # Content/Description
        lines.append("## Content:")
        content = item.get('content', item.get('description', ''))
        if content:
            lines.append(content)
        else:
            lines.append("No content available")
        lines.append("")
        
        return "\n".join(lines)

    # Implement abstract methods from BaseScraper
    def fetch_content(self) -> List[Dict[str, Any]]:
        """Fetch content for regular BaseScraper compatibility."""
        # For competitive scrapers, we mainly use run_backlog_capture and run_incremental_sync
        # This method provides compatibility with the base class
        return self.discover_content_urls(10)  # Get latest 10 items

    def get_incremental_items(self, items: List[Dict[str, Any]], state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Get only new items since last sync."""
        known_urls = state.get('content_urls', set())
        
        new_items = []
        for item in items:
            item_url = item.get('url')
            if item_url and item_url not in known_urls:
                new_items.append(item)
        
        return new_items

    def update_state(self, state: Dict[str, Any], items: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Update state with new items."""
        if 'content_urls' not in state:
            state['content_urls'] = set()
        
        for item in items:
            if 'url' in item:
                state['content_urls'].add(item['url'])
        
        state['last_update'] = datetime.now(self.tz).isoformat()
        state['last_item_count'] = len(items)
        
        return state