#!/usr/bin/env python3 """ Production script for API-based content scraping Captures YouTube videos and MailChimp campaigns using official APIs """ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from src.youtube_api_scraper import YouTubeAPIScraper from src.mailchimp_api_scraper import MailChimpAPIScraper from src.base_scraper import ScraperConfig from datetime import datetime import pytz import time import logging # Set up logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('logs/api_scrapers_production.log'), logging.StreamHandler() ] ) logger = logging.getLogger('api_production') def run_youtube_api_production(): """Run YouTube API scraper for production backlog""" logger.info("=" * 60) logger.info("YOUTUBE API SCRAPER - PRODUCTION RUN") logger.info("=" * 60) tz = pytz.timezone('America/Halifax') timestamp = datetime.now(tz).strftime('%Y-%m-%dT%H%M%S') config = ScraperConfig( source_name='youtube', brand_name='hvacknowitall', data_dir=Path('data/youtube'), logs_dir=Path('logs/youtube'), timezone='America/Halifax' ) try: scraper = YouTubeAPIScraper(config) logger.info("Starting YouTube API fetch for full channel...") start = time.time() # Fetch all videos with transcripts for top 50 videos = scraper.fetch_content(fetch_transcripts=True) elapsed = time.time() - start logger.info(f"Fetched {len(videos)} videos in {elapsed:.1f} seconds") if videos: # Statistics total_views = sum(v.get('view_count', 0) for v in videos) total_likes = sum(v.get('like_count', 0) for v in videos) with_transcripts = sum(1 for v in videos if v.get('transcript')) logger.info(f"Statistics:") logger.info(f" Total videos: {len(videos)}") logger.info(f" Total views: {total_views:,}") logger.info(f" Total likes: {total_likes:,}") logger.info(f" Videos with transcripts: {with_transcripts}") logger.info(f" Quota used: {scraper.quota_used}/{scraper.daily_quota_limit} units") # Save markdown with timestamp markdown = scraper.format_markdown(videos) output_file = Path(f'data/youtube/hvacknowitall_youtube_{timestamp}.md') output_file.parent.mkdir(parents=True, exist_ok=True) output_file.write_text(markdown, encoding='utf-8') logger.info(f"Markdown saved to: {output_file}") # Also save as "latest" for easy access latest_file = Path('data/youtube/hvacknowitall_youtube_latest.md') latest_file.write_text(markdown, encoding='utf-8') logger.info(f"Latest file updated: {latest_file}") # Update state file state = scraper.load_state() state = scraper.update_state(state, videos) scraper.save_state(state) logger.info("State file updated for incremental updates") return True, len(videos), output_file else: logger.error("No videos fetched from YouTube API") return False, 0, None except Exception as e: logger.error(f"YouTube API scraper failed: {e}") return False, 0, None def run_mailchimp_api_production(): """Run MailChimp API scraper for production backlog""" logger.info("\n" + "=" * 60) logger.info("MAILCHIMP API SCRAPER - PRODUCTION RUN") logger.info("=" * 60) tz = pytz.timezone('America/Halifax') timestamp = datetime.now(tz).strftime('%Y-%m-%dT%H%M%S') config = ScraperConfig( source_name='mailchimp', brand_name='hvacknowitall', data_dir=Path('data/mailchimp'), logs_dir=Path('logs/mailchimp'), timezone='America/Halifax' ) try: scraper = MailChimpAPIScraper(config) logger.info("Starting MailChimp API fetch for all campaigns...") start = time.time() # Fetch all campaigns from Bi-Weekly Newsletter folder campaigns = scraper.fetch_content(max_items=1000) # Get all available elapsed = time.time() - start logger.info(f"Fetched {len(campaigns)} campaigns in {elapsed:.1f} seconds") if campaigns: # Statistics total_sent = sum(c.get('metrics', {}).get('emails_sent', 0) for c in campaigns) total_opens = sum(c.get('metrics', {}).get('unique_opens', 0) for c in campaigns) total_clicks = sum(c.get('metrics', {}).get('unique_clicks', 0) for c in campaigns) logger.info(f"Statistics:") logger.info(f" Total campaigns: {len(campaigns)}") logger.info(f" Total emails sent: {total_sent:,}") logger.info(f" Total unique opens: {total_opens:,}") logger.info(f" Total unique clicks: {total_clicks:,}") if campaigns: avg_open_rate = sum(c.get('metrics', {}).get('open_rate', 0) for c in campaigns) / len(campaigns) avg_click_rate = sum(c.get('metrics', {}).get('click_rate', 0) for c in campaigns) / len(campaigns) logger.info(f" Average open rate: {avg_open_rate*100:.1f}%") logger.info(f" Average click rate: {avg_click_rate*100:.1f}%") # Save markdown with timestamp markdown = scraper.format_markdown(campaigns) output_file = Path(f'data/mailchimp/hvacknowitall_mailchimp_{timestamp}.md') output_file.parent.mkdir(parents=True, exist_ok=True) output_file.write_text(markdown, encoding='utf-8') logger.info(f"Markdown saved to: {output_file}") # Also save as "latest" for easy access latest_file = Path('data/mailchimp/hvacknowitall_mailchimp_latest.md') latest_file.write_text(markdown, encoding='utf-8') logger.info(f"Latest file updated: {latest_file}") # Update state file state = scraper.load_state() state = scraper.update_state(state, campaigns) scraper.save_state(state) logger.info("State file updated for incremental updates") return True, len(campaigns), output_file else: logger.warning("No campaigns found in MailChimp") return True, 0, None # Not an error if no campaigns except Exception as e: logger.error(f"MailChimp API scraper failed: {e}") return False, 0, None def sync_to_nas(): """Sync API scraper results to NAS""" logger.info("\n" + "=" * 60) logger.info("SYNCING TO NAS") logger.info("=" * 60) import subprocess nas_base = Path('/mnt/nas/hvacknowitall') # Sync YouTube try: youtube_src = Path('data/youtube') youtube_dest = nas_base / 'markdown_current/youtube' if youtube_src.exists() and any(youtube_src.glob('*.md')): # Create destination if needed youtube_dest.mkdir(parents=True, exist_ok=True) # Sync markdown files cmd = ['rsync', '-av', '--include=*.md', '--exclude=*', str(youtube_src) + '/', str(youtube_dest) + '/'] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: logger.info(f"✅ YouTube data synced to NAS: {youtube_dest}") else: logger.warning(f"YouTube sync warning: {result.stderr}") else: logger.info("No YouTube data to sync") except Exception as e: logger.error(f"Failed to sync YouTube data: {e}") # Sync MailChimp try: mailchimp_src = Path('data/mailchimp') mailchimp_dest = nas_base / 'markdown_current/mailchimp' if mailchimp_src.exists() and any(mailchimp_src.glob('*.md')): # Create destination if needed mailchimp_dest.mkdir(parents=True, exist_ok=True) # Sync markdown files cmd = ['rsync', '-av', '--include=*.md', '--exclude=*', str(mailchimp_src) + '/', str(mailchimp_dest) + '/'] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: logger.info(f"✅ MailChimp data synced to NAS: {mailchimp_dest}") else: logger.warning(f"MailChimp sync warning: {result.stderr}") else: logger.info("No MailChimp data to sync") except Exception as e: logger.error(f"Failed to sync MailChimp data: {e}") def main(): """Main production run""" logger.info("=" * 60) logger.info("HVAC KNOW IT ALL - API SCRAPERS PRODUCTION RUN") logger.info("=" * 60) logger.info(f"Started at: {datetime.now(pytz.timezone('America/Halifax')).isoformat()}") # Track results results = { 'youtube': {'success': False, 'count': 0, 'file': None}, 'mailchimp': {'success': False, 'count': 0, 'file': None} } # Run YouTube API scraper success, count, output_file = run_youtube_api_production() results['youtube'] = {'success': success, 'count': count, 'file': output_file} # Run MailChimp API scraper success, count, output_file = run_mailchimp_api_production() results['mailchimp'] = {'success': success, 'count': count, 'file': output_file} # Sync to NAS sync_to_nas() # Summary logger.info("\n" + "=" * 60) logger.info("PRODUCTION RUN SUMMARY") logger.info("=" * 60) for source, result in results.items(): status = "✅" if result['success'] else "❌" logger.info(f"{status} {source.upper()}: {result['count']} items") if result['file']: logger.info(f" Output: {result['file']}") logger.info(f"\nCompleted at: {datetime.now(pytz.timezone('America/Halifax')).isoformat()}") # Return success if at least one scraper succeeded return any(r['success'] for r in results.values()) if __name__ == "__main__": success = main() sys.exit(0 if success else 1)