import os
import re
import time
import json
import xml.etree.ElementTree as ET
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urljoin, urlparse
from scrapling import StealthyFetcher

from .base_competitive_scraper import BaseCompetitiveScraper, CompetitiveConfig


class HVACRSchoolCompetitiveScraper(BaseCompetitiveScraper):
    """Competitive intelligence scraper for HVACR School content."""
    
    def __init__(self, data_dir: Path, logs_dir: Path):
        """Initialize HVACR School competitive scraper."""
        config = CompetitiveConfig(
            source_name="hvacrschool_competitive",
            brand_name="hkia",
            competitor_name="hvacrschool",
            base_url="https://hvacrschool.com",
            data_dir=data_dir,
            logs_dir=logs_dir,
            request_delay=3.0,  # Conservative delay for competitor scraping
            backlog_limit=100,
            use_proxy=True
        )
        
        super().__init__(config)
        
        # HVACR School specific URLs
        self.sitemap_url = "https://hvacrschool.com/sitemap-1.xml"
        self.blog_base_url = "https://hvacrschool.com"
        
        # Initialize scrapling for advanced bot detection avoidance
        try:
            self.scraper = StealthyFetcher(
                headless=True,  # Use headless for production
                stealth_mode=True,
                block_images=True,  # Faster loading
                block_css=True,
                timeout=30
            )
            self.logger.info("Initialized StealthyFetcher for HVACR School competitive scraping")
        except Exception as e:
            self.logger.warning(f"Failed to initialize StealthyFetcher: {e}. Will use standard requests.")
            self.scraper = None
        
        # Content patterns specific to HVACR School
        self.content_selectors = [
            'article',
            '.entry-content',
            '.post-content',
            '.content',
            'main .content',
            '[role="main"]'
        ]
        
        # Patterns to identify article URLs vs pages/categories
        self.article_url_patterns = [
            r'^https?://hvacrschool\.com/[^/]+/?$',  # Direct articles
            r'^https?://hvacrschool\.com/[\w-]+/?$'  # Word-based article slugs
        ]
        
        self.skip_url_patterns = [
            '/page/', '/category/', '/tag/', '/author/',
            '/feed', '/wp-', '/search', '.xml', '.txt',
            '/partners/', '/resources/', '/content/',
            '/events/', '/jobs/', '/contact/', '/about/',
            '/privacy/', '/terms/', '/disclaimer/',
            '/subscribe/', '/newsletter/', '/login/'
        ]

    def discover_content_urls(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
        """Discover HVACR School content URLs from sitemap and recent posts."""
        self.logger.info(f"Discovering HVACR School content URLs (limit: {limit})")
        
        urls = []
        
        # Method 1: Sitemap discovery
        sitemap_urls = self._discover_from_sitemap()
        urls.extend(sitemap_urls)
        
        # Method 2: Recent posts discovery (if sitemap fails or is incomplete)
        if len(urls) < 10:  # Fallback if sitemap didn't yield enough URLs
            recent_urls = self._discover_recent_posts()
            urls.extend(recent_urls)
        
        # Remove duplicates while preserving order
        seen = set()
        unique_urls = []
        for url_data in urls:
            url = url_data['url']
            if url not in seen:
                seen.add(url)
                unique_urls.append(url_data)
        
        # Apply limit
        if limit:
            unique_urls = unique_urls[:limit]
        
        # Sort by last modified date (newest first)
        unique_urls.sort(key=lambda x: x.get('lastmod', ''), reverse=True)
        
        self.logger.info(f"Discovered {len(unique_urls)} unique HVACR School URLs")
        return unique_urls

    def _discover_from_sitemap(self) -> List[Dict[str, Any]]:
        """Discover URLs from HVACR School sitemap."""
        self.logger.info("Discovering URLs from HVACR School sitemap")
        
        try:
            response = self.make_competitive_request(self.sitemap_url)
            response.raise_for_status()
            
            # Parse XML sitemap
            root = ET.fromstring(response.content)
            namespaces = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
            
            urls = []
            for url_elem in root.findall('.//ns:url', namespaces):
                loc_elem = url_elem.find('ns:loc', namespaces)
                lastmod_elem = url_elem.find('ns:lastmod', namespaces)
                
                if loc_elem is not None:
                    url = loc_elem.text
                    lastmod = lastmod_elem.text if lastmod_elem is not None else None
                    
                    if self._is_article_url(url):
                        urls.append({
                            'url': url,
                            'lastmod': lastmod,
                            'discovery_method': 'sitemap'
                        })
            
            self.logger.info(f"Found {len(urls)} article URLs in sitemap")
            return urls
            
        except Exception as e:
            self.logger.error(f"Error discovering URLs from sitemap: {e}")
            return []

    def _discover_recent_posts(self) -> List[Dict[str, Any]]:
        """Discover recent posts from main blog page and pagination."""
        self.logger.info("Discovering recent HVACR School posts")
        
        urls = []
        
        try:
            # Try to find blog listing pages
            blog_urls = [
                "https://hvacrschool.com",
                "https://hvacrschool.com/blog",
                "https://hvacrschool.com/articles"
            ]
            
            for blog_url in blog_urls:
                try:
                    self.logger.debug(f"Checking blog URL: {blog_url}")
                    
                    if self.scraper:
                        # Use scrapling for better content extraction
                        response = self.scraper.fetch(blog_url)
                        if response:
                            links = response.css('a[href*="hvacrschool.com"]')
                            for link in links:
                                href = str(link)
                                # Extract href attribute
                                href_match = re.search(r'href=["\']([^"\']+)["\']', href)
                                if href_match:
                                    url = href_match.group(1)
                                    if self._is_article_url(url):
                                        urls.append({
                                            'url': url,
                                            'discovery_method': 'blog_listing'
                                        })
                    else:
                        # Fallback to standard requests
                        response = self.make_competitive_request(blog_url)
                        response.raise_for_status()
                        
                        # Extract article links using regex
                        article_links = re.findall(
                            r'href=["\']([^"\']+)["\']',
                            response.text
                        )
                        
                        for link in article_links:
                            if self._is_article_url(link):
                                urls.append({
                                    'url': link,
                                    'discovery_method': 'blog_listing'
                                })
                    
                    # If we found URLs from this source, we can stop
                    if urls:
                        break
                        
                except Exception as e:
                    self.logger.debug(f"Failed to discover from {blog_url}: {e}")
                    continue
            
            # Remove duplicates
            unique_urls = []
            seen = set()
            for url_data in urls:
                url = url_data['url']
                if url not in seen:
                    seen.add(url)
                    unique_urls.append(url_data)
            
            self.logger.info(f"Discovered {len(unique_urls)} URLs from blog listings")
            return unique_urls
            
        except Exception as e:
            self.logger.error(f"Error discovering recent posts: {e}")
            return []

    def _is_article_url(self, url: str) -> bool:
        """Determine if URL is an HVACR School article."""
        if not url:
            return False
        
        # Normalize URL
        url = url.strip()
        if not url.startswith(('http://', 'https://')):
            if url.startswith('/'):
                url = self.blog_base_url + url
            else:
                url = self.blog_base_url + '/' + url
        
        # Check skip patterns first
        for pattern in self.skip_url_patterns:
            if pattern in url:
                return False
        
        # Must be from HVACR School domain
        parsed = urlparse(url)
        if parsed.netloc not in ['hvacrschool.com', 'www.hvacrschool.com']:
            return False
        
        # Check against article patterns
        for pattern in self.article_url_patterns:
            if re.match(pattern, url):
                return True
        
        # Additional heuristics
        path = parsed.path.strip('/')
        if path and '/' not in path and len(path) > 3:
            # Single-level path likely an article
            return True
        
        return False

    def scrape_content_item(self, url: str) -> Optional[Dict[str, Any]]:
        """Scrape individual HVACR School content item."""
        self.logger.debug(f"Scraping HVACR School content: {url}")
        
        # Check cache first
        if url in self.content_cache:
            return self.content_cache[url]
        
        try:
            # Try Jina AI extraction first (if available)
            jina_result = self.extract_with_jina(url)
            if jina_result and jina_result.get('content'):
                content_data = self._parse_jina_content(jina_result['content'], url)
                if content_data:
                    content_data['extraction_method'] = 'jina_ai'
                    content_data['capture_timestamp'] = datetime.now(self.tz).isoformat()
                    self.content_cache[url] = content_data
                    return content_data
            
            # Fallback to direct scraping
            return self._scrape_with_scrapling(url)
            
        except Exception as e:
            self.logger.error(f"Error scraping HVACR School content {url}: {e}")
            return None

    def _parse_jina_content(self, jina_content: str, url: str) -> Optional[Dict[str, Any]]:
        """Parse content extracted by Jina AI."""
        try:
            lines = jina_content.split('\n')
            
            # Extract title (usually the first heading)
            title = "Untitled"
            for line in lines:
                line = line.strip()
                if line.startswith('# '):
                    title = line[2:].strip()
                    break
            
            # Extract main content (everything after title processing)
            content_lines = []
            skip_next = False
            
            for i, line in enumerate(lines):
                line = line.strip()
                
                if skip_next:
                    skip_next = False
                    continue
                
                # Skip navigation and metadata
                if any(skip_text in line.lower() for skip_text in [
                    'share this', 'facebook', 'twitter', 'linkedin',
                    'subscribe', 'newsletter', 'podcast',
                    'previous episode', 'next episode'
                ]):
                    continue
                
                # Include substantial content
                if len(line) > 20 or line.startswith(('#', '*', '-', '1.', '2.')):
                    content_lines.append(line)
            
            content = '\n'.join(content_lines).strip()
            
            # Extract basic metadata
            word_count = len(content.split()) if content else 0
            
            # Generate article ID
            import hashlib
            article_id = hashlib.md5(url.encode()).hexdigest()[:12]
            
            return {
                'id': article_id,
                'title': title,
                'url': url,
                'content': content,
                'word_count': word_count,
                'author': 'HVACR School',
                'type': 'blog_post',
                'source': 'hvacrschool',
                'categories': ['HVAC', 'Technical Education']
            }
            
        except Exception as e:
            self.logger.error(f"Error parsing Jina content for {url}: {e}")
            return None

    def _scrape_with_scrapling(self, url: str) -> Optional[Dict[str, Any]]:
        """Scrape HVACR School content using scrapling."""
        if not self.scraper:
            return self._scrape_with_requests(url)
        
        try:
            response = self.scraper.fetch(url)
            if not response:
                return None
            
            # Extract title
            title = "Untitled"
            title_selectors = ['h1', 'title', '.entry-title', '.post-title']
            for selector in title_selectors:
                title_elem = response.css_first(selector)
                if title_elem:
                    title = str(title_elem)
                    # Clean HTML tags
                    title = re.sub(r'<[^>]+>', '', title).strip()
                    if title:
                        break
            
            # Extract main content
            content = ""
            for selector in self.content_selectors:
                content_elem = response.css_first(selector)
                if content_elem:
                    content = str(content_elem)
                    break
            
            # Clean content
            if content:
                content = self._clean_hvacr_school_content(content)
            
            # Extract metadata
            author = "HVACR School"
            publish_date = None
            
            # Try to extract publish date
            date_selectors = [
                'meta[property="article:published_time"]',
                'meta[name="pubdate"]',
                '.published',
                '.date'
            ]
            
            for selector in date_selectors:
                date_elem = response.css_first(selector)
                if date_elem:
                    date_str = str(date_elem)
                    # Extract content attribute or text
                    if 'content="' in date_str:
                        start = date_str.find('content="') + 9
                        end = date_str.find('"', start)
                        if end > start:
                            publish_date = date_str[start:end]
                            break
                    else:
                        date_text = re.sub(r'<[^>]+>', '', date_str).strip()
                        if date_text and len(date_text) < 50:  # Reasonable date length
                            publish_date = date_text
                            break
            
            # Generate article ID and calculate metrics
            import hashlib
            article_id = hashlib.md5(url.encode()).hexdigest()[:12]
            
            content_text = re.sub(r'<[^>]+>', '', content) if content else ""
            word_count = len(content_text.split()) if content_text else 0
            
            result = {
                'id': article_id,
                'title': title,
                'url': url,
                'content': content,
                'author': author,
                'publish_date': publish_date,
                'word_count': word_count,
                'type': 'blog_post',
                'source': 'hvacrschool',
                'categories': ['HVAC', 'Technical Education'],
                'extraction_method': 'scrapling',
                'capture_timestamp': datetime.now(self.tz).isoformat()
            }
            
            self.content_cache[url] = result
            return result
            
        except Exception as e:
            self.logger.error(f"Error scraping with scrapling {url}: {e}")
            return self._scrape_with_requests(url)

    def _scrape_with_requests(self, url: str) -> Optional[Dict[str, Any]]:
        """Fallback scraping with standard requests."""
        try:
            response = self.make_competitive_request(url)
            response.raise_for_status()
            
            html_content = response.text
            
            # Extract title using regex
            title_match = re.search(r'<title[^>]*>(.*?)</title>', html_content, re.IGNORECASE | re.DOTALL)
            title = title_match.group(1).strip() if title_match else "Untitled"
            title = re.sub(r'<[^>]+>', '', title)
            
            # Extract main content using regex patterns
            content = ""
            content_patterns = [
                r'<article[^>]*>(.*?)</article>',
                r'<div[^>]*class="[^"]*entry-content[^"]*"[^>]*>(.*?)</div>',
                r'<div[^>]*class="[^"]*post-content[^"]*"[^>]*>(.*?)</div>',
                r'<main[^>]*>(.*?)</main>'
            ]
            
            for pattern in content_patterns:
                match = re.search(pattern, html_content, re.IGNORECASE | re.DOTALL)
                if match:
                    content = match.group(1)
                    break
            
            # Clean content
            if content:
                content = self._clean_hvacr_school_content(content)
            
            # Generate result
            import hashlib
            article_id = hashlib.md5(url.encode()).hexdigest()[:12]
            
            content_text = re.sub(r'<[^>]+>', '', content) if content else ""
            word_count = len(content_text.split()) if content_text else 0
            
            result = {
                'id': article_id,
                'title': title,
                'url': url,
                'content': content,
                'author': 'HVACR School',
                'word_count': word_count,
                'type': 'blog_post',
                'source': 'hvacrschool',
                'categories': ['HVAC', 'Technical Education'],
                'extraction_method': 'requests_regex',
                'capture_timestamp': datetime.now(self.tz).isoformat()
            }
            
            self.content_cache[url] = result
            return result
            
        except Exception as e:
            self.logger.error(f"Error scraping with requests {url}: {e}")
            return None

    def _clean_hvacr_school_content(self, content: str) -> str:
        """Clean HVACR School specific content."""
        try:
            # Remove common HVACR School specific elements
            remove_patterns = [
                # Podcast sections
                r'<div[^>]*class="[^"]*podcast[^"]*"[^>]*>.*?</div>',
                r'#### Our latest Podcast.*?(?=<h[1-6]|$)',
                r'Audio Player.*?(?=<h[1-6]|$)',
                
                # Social sharing
                r'<div[^>]*class="[^"]*share[^"]*"[^>]*>.*?</div>',
                r'Share this:.*?(?=<h[1-6]|$)',
                r'Share this Tech Tip:.*?(?=<h[1-6]|$)',
                
                # Navigation
                r'<nav[^>]*>.*?</nav>',
                r'<aside[^>]*>.*?</aside>',
                
                # Comments and related
                r'## Comments.*?(?=<h[1-6]|##|$)',
                r'## Related Tech Tips.*?(?=<h[1-6]|##|$)',
                
                # Footer and ads
                r'<footer[^>]*>.*?</footer>',
                r'<div[^>]*class="[^"]*ad[^"]*"[^>]*>.*?</div>',
                
                # Promotional content
                r'Subscribe to free tech tips\.',
                r'### Get Tech Tips.*?(?=<h[1-6]|##|$)',
            ]
            
            cleaned_content = content
            for pattern in remove_patterns:
                cleaned_content = re.sub(pattern, '', cleaned_content, flags=re.DOTALL | re.IGNORECASE)
            
            # Remove excessive whitespace
            cleaned_content = re.sub(r'\n\s*\n\s*\n+', '\n\n', cleaned_content)
            cleaned_content = re.sub(r'[ \t]+', ' ', cleaned_content)
            
            return cleaned_content.strip()
            
        except Exception as e:
            self.logger.warning(f"Error cleaning HVACR School content: {e}")
            return content

    def download_competitive_media(self, url: str, article_id: str) -> Optional[str]:
        """Download images from HVACR School content."""
        try:
            # Skip certain types of images that are not valuable for competitive intelligence
            skip_patterns = [
                'logo', 'icon', 'avatar', 'sponsor', 'ad',
                'social', 'share', 'button'
            ]
            
            url_lower = url.lower()
            if any(pattern in url_lower for pattern in skip_patterns):
                return None
            
            # Use base class media download with competitive directory
            media_dir = self.config.data_dir / "competitive_intelligence" / self.competitor_name / "media"
            media_dir.mkdir(parents=True, exist_ok=True)
            
            filename = f"hvacrschool_{article_id}_{int(time.time())}"
            
            # Determine file extension
            if url_lower.endswith(('.jpg', '.jpeg')):
                filename += '.jpg'
            elif url_lower.endswith('.png'):
                filename += '.png'
            elif url_lower.endswith('.gif'):
                filename += '.gif'
            else:
                filename += '.jpg'  # Default
            
            filepath = media_dir / filename
            
            # Download the image
            response = self.make_competitive_request(url, stream=True)
            response.raise_for_status()
            
            with open(filepath, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    f.write(chunk)
            
            self.logger.info(f"Downloaded competitive media: {filename}")
            return str(filepath)
            
        except Exception as e:
            self.logger.warning(f"Failed to download competitive media {url}: {e}")
            return None

    def __del__(self):
        """Clean up scrapling resources."""
        try:
            if hasattr(self, 'scraper') and self.scraper and hasattr(self.scraper, 'close'):
                self.scraper.close()
        except:
            pass