import os
import time
import re
import xml.etree.ElementTree as ET
from typing import Any, Dict, List, Optional
from datetime import datetime
from urllib.parse import urljoin, urlparse
from pathlib import Path
from scrapling import StealthyFetcher
from src.base_scraper import BaseScraper, ScraperConfig


class HVACRSchoolScraper(BaseScraper):
    """Scraper for HVACR School blog content using scrapling for anti-bot detection."""
    
    def __init__(self, config: ScraperConfig):
        super().__init__(config)
        self.base_url = "http://www.hvacrschool.com/"
        self.sitemap_url = "http://www.hvacrschool.com/sitemap-1.xml"
        
        # Initialize scrapling with anti-bot features
        self.scraper = StealthyFetcher(
            headless=False,  # Use headed browser to avoid detection
            # Note: StealthyFetcher automatically includes stealth mode
        )
        
        # Cache for parsed articles to avoid re-scraping
        self.article_cache = {}
        
        # Rate limiting settings
        self.request_delay = 2.0  # Seconds between requests
        self.last_request_time = 0

    def _apply_rate_limit(self):
        """Apply rate limiting between requests."""
        current_time = time.time()
        time_since_last = current_time - self.last_request_time
        if time_since_last < self.request_delay:
            sleep_time = self.request_delay - time_since_last
            time.sleep(sleep_time)
        self.last_request_time = time.time()

    def fetch_sitemap_urls(self) -> List[Dict[str, str]]:
        """Fetch all article URLs from the sitemap."""
        self.logger.info("Fetching sitemap URLs")
        
        try:
            self._apply_rate_limit()
            response = self.make_request('GET', self.sitemap_url, timeout=30)
            response.raise_for_status()
            
            # Parse XML sitemap
            root = ET.fromstring(response.content)
            
            # Handle XML namespaces
            namespaces = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
            
            urls = []
            for url_elem in root.findall('.//ns:url', namespaces):
                loc_elem = url_elem.find('ns:loc', namespaces)
                lastmod_elem = url_elem.find('ns:lastmod', namespaces)
                
                if loc_elem is not None:
                    url = loc_elem.text
                    lastmod = lastmod_elem.text if lastmod_elem is not None else None
                    
                    # Filter for blog posts (exclude pages, feeds, etc.)
                    if self._is_article_url(url):
                        urls.append({
                            'url': url,
                            'lastmod': lastmod
                        })
            
            self.logger.info(f"Found {len(urls)} article URLs in sitemap")
            return urls
            
        except Exception as e:
            self.logger.error(f"Error fetching sitemap: {e}")
            return []

    def _is_article_url(self, url: str) -> bool:
        """Determine if URL is an article based on patterns."""
        # Skip non-article URLs
        skip_patterns = [
            '/page/',
            '/category/',
            '/tag/',
            '/author/',
            '/feed',
            '/wp-',
            '/search',
            '.xml',
            '.txt',
            '/partners/',
            '/resources/',
            '/content/',
            '/events/',
            '/jobs/',
            '/contact/',
            '/about/',
            '/privacy/',
            '/terms/',
            '/disclaimer/',
        ]
        
        # Check if URL should be skipped
        for pattern in skip_patterns:
            if pattern in url:
                return False
        
        # Must be from the main domain
        parsed = urlparse(url)
        if parsed.netloc not in ['www.hvacrschool.com', 'hvacrschool.com']:
            return False
        
        # Should have a path with content (not just root)
        path = parsed.path.strip('/')
        if not path:
            return False
        
        # Additional check: should not end with just slash (likely a page, not article)
        if path.count('/') == 0 and not path.endswith('.html'):
            # This is likely an article URL like "understanding-heat-transfer"
            return True
        elif path.count('/') > 1:
            # This is likely a nested URL which might not be an article
            return False
        
        return True

    def scrape_article(self, url: str) -> Optional[Dict[str, Any]]:
        """Scrape a single article using scrapling."""
        if url in self.article_cache:
            return self.article_cache[url]
        
        try:
            self.logger.debug(f"Scraping article: {url}")
            self._apply_rate_limit()
            
            # Use scrapling to fetch the page
            response = self.scraper.fetch(url)
            
            if not response:
                self.logger.warning(f"No response for URL: {url}")
                return None
            
            # Extract article data
            article_data = self._extract_article_data(response, url)
            
            # Cache the result
            if article_data:
                self.article_cache[url] = article_data
            
            return article_data
            
        except Exception as e:
            self.logger.error(f"Error scraping article {url}: {e}")
            return None

    def _extract_article_data(self, response, url: str) -> Optional[Dict[str, Any]]:
        """Extract structured data from the article page."""
        try:
            # Try to extract JSON-LD structured data first
            json_ld_scripts = response.css('script[type="application/ld+json"]')
            structured_data = None
            
            for script in json_ld_scripts:
                try:
                    import json
                    script_text = str(script)
                    # Extract text between script tags
                    start = script_text.find('>') + 1
                    end = script_text.rfind('<')
                    if start > 0 and end > start:
                        json_text = script_text[start:end].strip()
                        data = json.loads(json_text)
                        if isinstance(data, dict) and data.get('@type') in ['Article', 'BlogPosting']:
                            structured_data = data
                            break
                except Exception as e:
                    self.logger.debug(f"Failed to parse JSON-LD: {e}")
                    continue
            
            # Extract title
            title = None
            if structured_data and 'headline' in structured_data:
                title = structured_data['headline']
            else:
                title_elem = response.css_first('h1') or response.css_first('title')
                if title_elem:
                    title = str(title_elem).replace('<h1>', '').replace('</h1>', '').replace('<title>', '').replace('</title>', '').strip()
            
            # Extract content with filtering
            content = ""
            content_selectors = [
                'article',
                '.entry-content',
                '.post-content', 
                '.content',
                'main'
            ]
            
            for selector in content_selectors:
                content_elem = response.css_first(selector)
                if content_elem:
                    content = str(content_elem)
                    break
            
            # Clean content by removing irrelevant sections
            if content:
                content = self._clean_article_content(content)
                content = self._download_content_images(content, self._generate_article_id(url), url)
            
            # Extract metadata
            author = "HVACR School"  # Default author
            if structured_data and 'author' in structured_data:
                author_data = structured_data['author']
                if isinstance(author_data, dict):
                    author = author_data.get('name', author)
                elif isinstance(author_data, str):
                    author = author_data
            
            # Extract publish date
            publish_date = None
            if structured_data and 'datePublished' in structured_data:
                publish_date = structured_data['datePublished']
            else:
                # Try to find date in meta tags
                date_meta = response.css_first('meta[property="article:published_time"]')
                if date_meta:
                    # Extract content attribute from meta tag
                    meta_str = str(date_meta)
                    if 'content="' in meta_str:
                        start = meta_str.find('content="') + 9
                        end = meta_str.find('"', start)
                        if end > start:
                            publish_date = meta_str[start:end]
            
            # Extract description/excerpt
            description = ""
            if structured_data and 'description' in structured_data:
                description = structured_data['description']
            else:
                # Try meta description
                meta_desc = response.css_first('meta[name="description"]')
                if meta_desc:
                    # Extract content attribute from meta tag
                    meta_str = str(meta_desc)
                    if 'content="' in meta_str:
                        start = meta_str.find('content="') + 9
                        end = meta_str.find('"', start)
                        if end > start:
                            description = meta_str[start:end]
            
            # Extract categories/tags
            categories = []
            if structured_data and 'keywords' in structured_data:
                keywords = structured_data['keywords']
                if isinstance(keywords, list):
                    categories = keywords
                elif isinstance(keywords, str):
                    categories = [k.strip() for k in keywords.split(',')]
            
            # Build article data
            article_data = {
                'id': self._generate_article_id(url),
                'title': title or 'Untitled',
                'url': url,
                'author': author,
                'publish_date': publish_date,
                'content': content,
                'description': description,
                'categories': categories,
                'type': 'blog_post',
                'source': 'hvacrschool'
            }
            
            # Calculate word count
            if content:
                text_content = self.convert_to_markdown(content)
                article_data['word_count'] = len(text_content.split())
            else:
                article_data['word_count'] = 0
            
            return article_data
            
        except Exception as e:
            self.logger.error(f"Error extracting article data from {url}: {e}")
            return None

    def _generate_article_id(self, url: str) -> str:
        """Generate a consistent ID from the URL."""
        import hashlib
        return hashlib.md5(url.encode()).hexdigest()[:12]
    
    def _clean_article_content(self, content: str) -> str:
        """Clean article content by removing irrelevant sections."""
        try:
            # Remove common irrelevant sections using regex patterns
            import re
            
            # Patterns for content to remove
            remove_patterns = [
                # Podcast sections
                r'<div[^>]*class="[^"]*podcast[^"]*"[^>]*>.*?</div>',
                r'<section[^>]*class="[^"]*podcast[^"]*"[^>]*>.*?</section>',
                r'#### Our latest Podcast.*?(?=<h[1-6]|$)',
                r'#### Check out our most recent video.*?(?=<h[1-6]|$)',
                r'Audio Player.*?(?=<h[1-6]|$)',
                
                # Social sharing widgets
                r'<div[^>]*class="[^"]*share[^"]*"[^>]*>.*?</div>',
                r'Share this:.*?(?=<h[1-6]|$)',
                r'Share this Tech Tip:.*?(?=<h[1-6]|$)',
                r'\[Facebook\].*?\[Tweet\].*?(?=<h[1-6]|\n\n|$)',
                
                # Navigation and sidebar content
                r'<nav[^>]*>.*?</nav>',
                r'<aside[^>]*>.*?</aside>',
                r'<div[^>]*class="[^"]*sidebar[^"]*"[^>]*>.*?</div>',
                r'<div[^>]*class="[^"]*navigation[^"]*"[^>]*>.*?</div>',
                
                # Episode lists and related content
                r'Search Episodes.*?(?=<h[1-6]|$)',
                r'#### Check our latest Tech Tips.*?(?=<h[1-6]|$)',
                r'Load More.*?(?=<h[1-6]|$)',
                r'Previous Episode.*?Next Episode.*?(?=<h[1-6]|$)',
                r'Show Episodes List.*?(?=<h[1-6]|$)',
                r'Show Podcast Information.*?(?=<h[1-6]|$)',
                
                # Tech tip lists and promotional content
                r'\[.*?\]\(http://www\.hvacrschool\.com/.*?\)\s*\[.*?\]\(http://www\.hvacrschool\.com/.*?\)\s*\[.*?\]\(http://www\.hvacrschool\.com/.*?\)',
                r'#### Nylog Blue Gasket.*?(?=<h[1-6]|$)',
                
                # Print and sharing buttons
                r'\[\!\[Print Friendly.*?\]\].*?(?=<h[1-6]|\n\n|$)',
                r'\[BACK\]\(/\)',
                
                # Tag sections and metadata
                r'\[#[^\]]+\]\([^)]+\)(\s*\[#[^\]]+\]\([^)]+\))*',
                r'## Comments.*?(?=<h[1-6]|##|\n\n---|\n\n#|$)',
                r'## Related Tech Tips.*?(?=<h[1-6]|##|\n\n---|\n\n#|$)',
                
                # Navigation breadcrumbs and login prompts
                r'To leave a comment.*?Log In.*?(?=<h[1-6]|\n\n|$)',
                r'\[Log In\]\([^)]+\)',
                r'\[Read more\]\([^)]+\)',
                
                # Footer content
                r'<footer[^>]*>.*?</footer>',
                r'<div[^>]*class="[^"]*footer[^"]*"[^>]*>.*?</div>',
                
                # Advertisement sections
                r'<div[^>]*class="[^"]*ad[^"]*"[^>]*>.*?</div>',
                r'<div[^>]*class="[^"]*advertisement[^"]*"[^>]*>.*?</div>',
                
                # Subscribe prompts and promotional text
                r'Subscribe to free tech tips\.',
                r'### Get Tech Tips.*?(?=<h[1-6]|##|$)',
            ]
            
            # Apply all removal patterns
            cleaned_content = content
            for pattern in remove_patterns:
                cleaned_content = re.sub(pattern, '', cleaned_content, flags=re.DOTALL | re.IGNORECASE)
            
            # Remove excessive whitespace
            cleaned_content = re.sub(r'\n\s*\n\s*\n+', '\n\n', cleaned_content)
            cleaned_content = re.sub(r'[ \t]+', ' ', cleaned_content)
            
            return cleaned_content.strip()
            
        except Exception as e:
            self.logger.warning(f"Error cleaning content: {e}")
            return content

    def _download_content_images(self, content: str, article_id: str, base_url: str) -> str:
        """Download images from content and replace URLs with local paths."""
        try:
            # Find all image URLs in the HTML content
            img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
            images = re.finditer(img_pattern, content, re.IGNORECASE)
            
            downloaded_count = 0
            for match in images:
                img_tag = match.group(0)
                img_url = match.group(1)
                
                # Convert relative URLs to absolute
                if img_url.startswith('//'):
                    img_url = 'https:' + img_url
                elif img_url.startswith('/'):
                    img_url = urljoin(base_url, img_url)
                elif not img_url.startswith(('http://', 'https://')):
                    img_url = urljoin(base_url, img_url)
                
                # Skip SVGs, icons, very small images, and repetitive sponsor content
                skip_patterns = [
                    '.svg', 'icon', 'logo', 'avatar', '1x1',
                    'nylog_blue.jpg',
                    'venom-pack-condenser',
                    'viper_pandrain_webt',
                    'navac_association',
                    'fast-stat-hvac-school',
                    'copeland.png',
                    'santa-fe.png',
                    'uei.png',
                    'untitled_design_3-1-768x768.jpg',  # Podcast thumbnail
                    'placeholder.png',
                    'placeholder.gif'
                ]
                if any(skip in img_url.lower() for skip in skip_patterns):
                    self.logger.debug(f"Skipping repetitive/sponsor image: {img_url}")
                    continue
                
                # Download the image
                local_path = self.download_media(img_url, f"hvacrschool_{article_id}_img_{downloaded_count}", "image")
                
                if local_path:
                    # Convert to relative path for markdown
                    try:
                        rel_path = Path(local_path).relative_to(self.config.data_dir)
                        # Replace the img src in content
                        new_img_tag = img_tag.replace(img_url, str(rel_path))
                        content = content.replace(img_tag, new_img_tag)
                        downloaded_count += 1
                        self.logger.info(f"Downloaded image {downloaded_count}: {Path(local_path).name}")
                    except ValueError:
                        # If relative path fails, use absolute path
                        new_img_tag = img_tag.replace(img_url, local_path)
                        content = content.replace(img_tag, new_img_tag)
                        downloaded_count += 1
                
                # Rate limiting for image downloads
                if downloaded_count > 0 and downloaded_count % 3 == 0:
                    time.sleep(1)  # Brief pause every 3 images
                    
            if downloaded_count > 0:
                self.logger.info(f"Downloaded {downloaded_count} images for article {article_id}")
                
            return content
            
        except Exception as e:
            self.logger.error(f"Error downloading images for article {article_id}: {e}")
            return content

    def fetch_content(self, max_items: Optional[int] = None) -> List[Dict[str, Any]]:
        """Fetch blog posts from HVACR School."""
        self.logger.info(f"Starting HVACR School content fetch (max_items: {max_items})")
        
        # Get all URLs from sitemap
        sitemap_urls = self.fetch_sitemap_urls()
        
        if not sitemap_urls:
            self.logger.warning("No URLs found in sitemap")
            return []
        
        # Limit the number of articles if specified
        if max_items:
            # Sort by last modified date (newest first)
            sitemap_urls.sort(key=lambda x: x.get('lastmod', ''), reverse=True)
            sitemap_urls = sitemap_urls[:max_items]
        
        articles = []
        total_urls = len(sitemap_urls)
        
        for i, url_data in enumerate(sitemap_urls, 1):
            url = url_data['url']
            self.logger.info(f"Processing article {i}/{total_urls}: {url}")
            
            article = self.scrape_article(url)
            if article:
                articles.append(article)
            
            # Progress logging
            if i % 10 == 0:
                self.logger.info(f"Processed {i}/{total_urls} articles")
        
        self.logger.info(f"Successfully fetched {len(articles)} articles")
        return articles

    def format_markdown(self, articles: List[Dict[str, Any]]) -> str:
        """Format articles as markdown."""
        markdown_sections = []
        
        for article in articles:
            section = []
            
            # ID
            section.append(f"# ID: {article.get('id', 'N/A')}")
            section.append("")
            
            # Title
            title = article.get('title', 'Untitled')
            section.append(f"## Title: {title}")
            section.append("")
            
            # Type
            section.append("## Type: blog_post")
            section.append("")
            
            # Author
            author = article.get('author', 'HVACR School')
            section.append(f"## Author: {author}")
            section.append("")
            
            # Publish Date
            date = article.get('publish_date', '')
            section.append(f"## Publish Date: {date}")
            section.append("")
            
            # Word Count
            word_count = article.get('word_count', 0)
            section.append(f"## Word Count: {word_count}")
            section.append("")
            
            # Categories/Tags
            categories = article.get('categories', [])
            if categories:
                categories_str = ', '.join(categories)
            else:
                categories_str = 'HVAC, Refrigeration'  # Default categories
            section.append(f"## Categories: {categories_str}")
            section.append("")
            
            # Permalink
            url = article.get('url', '')
            section.append(f"## Permalink: {url}")
            section.append("")
            
            # Description/Content
            section.append("## Description:")
            content = article.get('content', '')
            if content:
                content_md = self.convert_to_markdown(content)
                section.append(content_md)
            else:
                description = article.get('description', 'No content available')
                section.append(description)
            section.append("")
            
            # Separator
            section.append("-" * 50)
            section.append("")
            
            markdown_sections.append('\n'.join(section))
        
        return '\n'.join(markdown_sections)

    def get_incremental_items(self, items: List[Dict[str, Any]], state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Get only new articles since last sync."""
        if not state:
            return items
        
        last_sync_date = state.get('last_sync_date')
        if not last_sync_date:
            return items
        
        new_items = []
        for item in items:
            article_date = item.get('publish_date')
            if article_date and article_date > last_sync_date:
                new_items.append(item)
            elif not article_date:
                # Include items without dates to be safe
                new_items.append(item)
        
        return new_items

    def update_state(self, state: Dict[str, Any], items: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Update state with latest article information."""
        if not items:
            return state
        
        # Find the latest article by publish date
        latest_date = None
        for item in items:
            article_date = item.get('publish_date')
            if article_date:
                if not latest_date or article_date > latest_date:
                    latest_date = article_date
        
        if latest_date:
            state['last_sync_date'] = latest_date
        
        state['last_sync'] = datetime.now(self.tz).isoformat()
        state['article_count'] = len(items)
        
        return state

    def __del__(self):
        """Clean up scrapling resources."""
        try:
            if hasattr(self, 'scraper') and hasattr(self.scraper, 'close'):
                self.scraper.close()
        except:
            pass