#!/usr/bin/env python3
"""
HTML Dashboard Generator for HVAC Know It All Content Aggregation System
Generates a web-based dashboard showing:
- System health overview
- Scraper performance metrics
- Resource usage trends
- Alert history
- Data collection statistics
"""
import json
import os
from pathlib import Path
from datetime import datetime, timedelta
from typing import Dict, List, Any
import logging
logger = logging.getLogger(__name__)
class DashboardGenerator:
"""Generate HTML dashboard from monitoring data"""
def __init__(self, monitoring_dir: Path = None):
self.monitoring_dir = monitoring_dir or Path("/opt/hvac-kia-content/monitoring")
self.metrics_dir = self.monitoring_dir / "metrics"
self.alerts_dir = self.monitoring_dir / "alerts"
self.dashboard_dir = self.monitoring_dir / "dashboard"
# Create dashboard directory
self.dashboard_dir.mkdir(parents=True, exist_ok=True)
def load_recent_metrics(self, metric_type: str, hours: int = 24) -> List[Dict[str, Any]]:
"""Load recent metrics of specified type"""
cutoff_time = datetime.now() - timedelta(hours=hours)
metrics = []
pattern = f"{metric_type}_*.json"
for metrics_file in sorted(self.metrics_dir.glob(pattern)):
try:
file_time = datetime.fromtimestamp(metrics_file.stat().st_mtime)
if file_time >= cutoff_time:
with open(metrics_file) as f:
data = json.load(f)
data['file_timestamp'] = file_time.isoformat()
metrics.append(data)
except Exception as e:
logger.warning(f"Error loading {metrics_file}: {e}")
return metrics
def load_recent_alerts(self, hours: int = 72) -> List[Dict[str, Any]]:
"""Load recent alerts"""
cutoff_time = datetime.now() - timedelta(hours=hours)
all_alerts = []
for alerts_file in sorted(self.alerts_dir.glob("alerts_*.json")):
try:
file_time = datetime.fromtimestamp(alerts_file.stat().st_mtime)
if file_time >= cutoff_time:
with open(alerts_file) as f:
alerts = json.load(f)
if isinstance(alerts, list):
all_alerts.extend(alerts)
else:
all_alerts.append(alerts)
except Exception as e:
logger.warning(f"Error loading {alerts_file}: {e}")
# Sort by timestamp
all_alerts.sort(key=lambda x: x.get('timestamp', ''), reverse=True)
return all_alerts
def generate_system_charts_js(self, system_metrics: List[Dict[str, Any]]) -> str:
"""Generate JavaScript for system resource charts"""
if not system_metrics:
return ""
# Extract data for charts
timestamps = []
cpu_data = []
memory_data = []
disk_data = []
for metric in system_metrics[-50:]: # Last 50 data points
if 'system' in metric and 'timestamp' in metric:
timestamp = metric['timestamp'][:16] # YYYY-MM-DDTHH:MM
timestamps.append(f"'{timestamp}'")
sys_data = metric['system']
cpu_data.append(sys_data.get('cpu_percent', 0))
memory_data.append(sys_data.get('memory_percent', 0))
disk_data.append(sys_data.get('disk_percent', 0))
return f"""
// System Resource Charts
const systemTimestamps = [{', '.join(timestamps)}];
const cpuData = {cpu_data};
const memoryData = {memory_data};
const diskData = {disk_data};
// CPU Chart
const cpuCtx = document.getElementById('cpuChart').getContext('2d');
new Chart(cpuCtx, {{
type: 'line',
data: {{
labels: systemTimestamps,
datasets: [{{
label: 'CPU Usage (%)',
data: cpuData,
borderColor: 'rgb(255, 99, 132)',
backgroundColor: 'rgba(255, 99, 132, 0.2)',
tension: 0.1
}}]
}},
options: {{
responsive: true,
scales: {{
y: {{
beginAtZero: true,
max: 100
}}
}}
}}
}});
// Memory Chart
const memoryCtx = document.getElementById('memoryChart').getContext('2d');
new Chart(memoryCtx, {{
type: 'line',
data: {{
labels: systemTimestamps,
datasets: [{{
label: 'Memory Usage (%)',
data: memoryData,
borderColor: 'rgb(54, 162, 235)',
backgroundColor: 'rgba(54, 162, 235, 0.2)',
tension: 0.1
}}]
}},
options: {{
responsive: true,
scales: {{
y: {{
beginAtZero: true,
max: 100
}}
}}
}}
}});
// Disk Chart
const diskCtx = document.getElementById('diskChart').getContext('2d');
new Chart(diskCtx, {{
type: 'line',
data: {{
labels: systemTimestamps,
datasets: [{{
label: 'Disk Usage (%)',
data: diskData,
borderColor: 'rgb(255, 205, 86)',
backgroundColor: 'rgba(255, 205, 86, 0.2)',
tension: 0.1
}}]
}},
options: {{
responsive: true,
scales: {{
y: {{
beginAtZero: true,
max: 100
}}
}}
}}
}});
"""
def generate_scraper_charts_js(self, app_metrics: List[Dict[str, Any]]) -> str:
"""Generate JavaScript for scraper performance charts"""
if not app_metrics:
return ""
# Collect scraper data over time
scraper_data = {}
timestamps = []
for metric in app_metrics[-20:]: # Last 20 data points
if 'scrapers' in metric and 'timestamp' in metric:
timestamp = metric['timestamp'][:16] # YYYY-MM-DDTHH:MM
if timestamp not in timestamps:
timestamps.append(timestamp)
for scraper_name, scraper_info in metric['scrapers'].items():
if scraper_name not in scraper_data:
scraper_data[scraper_name] = []
scraper_data[scraper_name].append(scraper_info.get('last_item_count', 0))
# Generate datasets for each scraper
datasets = []
colors = [
'rgb(255, 99, 132)', 'rgb(54, 162, 235)', 'rgb(255, 205, 86)',
'rgb(75, 192, 192)', 'rgb(153, 102, 255)', 'rgb(255, 159, 64)'
]
for i, (scraper_name, data) in enumerate(scraper_data.items()):
color = colors[i % len(colors)]
datasets.append(f"""{{
label: '{scraper_name}',
data: {data[-len(timestamps):]},
borderColor: '{color}',
backgroundColor: '{color.replace("rgb", "rgba").replace(")", ", 0.2)")}',
tension: 0.1
}}""")
return f"""
// Scraper Performance Chart
const scraperTimestamps = {[f"'{ts}'" for ts in timestamps]};
const scraperCtx = document.getElementById('scraperChart').getContext('2d');
new Chart(scraperCtx, {{
type: 'line',
data: {{
labels: scraperTimestamps,
datasets: [{', '.join(datasets)}]
}},
options: {{
responsive: true,
scales: {{
y: {{
beginAtZero: true
}}
}}
}}
}});
"""
def generate_html_dashboard(self, system_metrics: List[Dict[str, Any]],
app_metrics: List[Dict[str, Any]],
alerts: List[Dict[str, Any]]) -> str:
"""Generate complete HTML dashboard"""
# Get latest metrics for current status
latest_system = system_metrics[-1] if system_metrics else {}
latest_app = app_metrics[-1] if app_metrics else {}
# Calculate health status
critical_alerts = [a for a in alerts if a.get('type') == 'CRITICAL']
warning_alerts = [a for a in alerts if a.get('type') == 'WARNING']
if critical_alerts:
health_status = "CRITICAL"
health_color = "#dc3545" # Red
elif warning_alerts:
health_status = "WARNING"
health_color = "#ffc107" # Yellow
else:
health_status = "HEALTHY"
health_color = "#28a745" # Green
# Generate system status cards
system_cards = ""
if 'system' in latest_system:
sys_data = latest_system['system']
system_cards = f"""
CPU Usage
{sys_data.get('cpu_percent', 'N/A'):.1f}%
Memory Usage
{sys_data.get('memory_percent', 'N/A'):.1f}%
Disk Usage
{sys_data.get('disk_percent', 'N/A'):.1f}%
Uptime
{sys_data.get('uptime_hours', 0):.1f}h
"""
# Generate scraper status table
scraper_rows = ""
if 'scrapers' in latest_app:
for name, data in latest_app['scrapers'].items():
last_count = data.get('last_item_count', 0)
minutes_since = data.get('minutes_since_update')
if minutes_since is not None:
if minutes_since < 60:
time_str = f"{minutes_since:.0f}m ago"
status_color = "success"
elif minutes_since < 1440: # 24 hours
time_str = f"{minutes_since/60:.1f}h ago"
status_color = "warning"
else:
time_str = f"{minutes_since/1440:.1f}d ago"
status_color = "danger"
else:
time_str = "Never"
status_color = "secondary"
scraper_rows += f"""
{name.title()}
{last_count}
{time_str}
{data.get('last_id', 'N/A')}
"""
# Generate alerts table
alert_rows = ""
for alert in alerts[:10]: # Show last 10 alerts
alert_type = alert.get('type', 'INFO')
if alert_type == 'CRITICAL':
badge_class = "bg-danger"
elif alert_type == 'WARNING':
badge_class = "bg-warning"
else:
badge_class = "bg-info"
timestamp = alert.get('timestamp', '')[:19].replace('T', ' ')
alert_rows += f"""
{timestamp}
{alert_type}
{alert.get('component', 'N/A')}
{alert.get('message', 'N/A')}
"""
# Generate JavaScript for charts
system_charts_js = self.generate_system_charts_js(system_metrics)
scraper_charts_js = self.generate_scraper_charts_js(app_metrics)
html = f"""
HVAC Know It All - System Dashboard
System Status: {health_status}
Total Alerts: {len(alerts)} | Critical: {len(critical_alerts)} | Warnings: {len(warning_alerts)}
System Resources
{system_cards}
CPU Usage Trend
Memory Usage Trend
Disk Usage Trend
Scraper Item Collection Trend
Scraper Status
Scraper
Last Items
Last Update
Last ID
{scraper_rows}
Recent Alerts
Timestamp
Type
Component
Message
{alert_rows}
Dashboard auto-refreshes every 5 minutes.
Refresh Now
"""
return html
def generate_dashboard(self):
"""Generate and save the HTML dashboard"""
logger.info("Generating HTML dashboard...")
# Load recent metrics and alerts
system_metrics = self.load_recent_metrics('system', 24)
app_metrics = self.load_recent_metrics('application', 24)
alerts = self.load_recent_alerts(72)
# Generate HTML
html_content = self.generate_html_dashboard(system_metrics, app_metrics, alerts)
# Save dashboard
dashboard_file = self.dashboard_dir / "index.html"
try:
with open(dashboard_file, 'w') as f:
f.write(html_content)
logger.info(f"Dashboard saved to {dashboard_file}")
# Also create a timestamped version
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
backup_file = self.dashboard_dir / f"dashboard_{timestamp}.html"
with open(backup_file, 'w') as f:
f.write(html_content)
return dashboard_file
except Exception as e:
logger.error(f"Error saving dashboard: {e}")
return None
def main():
"""Generate dashboard"""
generator = DashboardGenerator()
dashboard_file = generator.generate_dashboard()
if dashboard_file:
print(f"Dashboard generated: {dashboard_file}")
print(f"View at: file://{dashboard_file.absolute()}")
return True
else:
print("Failed to generate dashboard")
return False
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
success = main()
exit(0 if success else 1)