/** * Performance Monitor - Phase 3 Integration * * Comprehensive performance monitoring system for E2E test integration validation. * Tracks resource utilization, concurrent load testing, performance benchmarking, * and optimization recommendations across all agent implementations. * * Features: * - Real-time resource monitoring (memory, CPU, network) * - Concurrent user simulation and load testing * - Performance benchmarking against targets * - Browser resource utilization tracking * - Network request analysis * - Page load time optimization * * @package HVAC_Community_Events * @version 3.0.0 * @created 2025-08-27 */ const fs = require('fs').promises; const path = require('path'); const { performance, PerformanceObserver } = require('perf_hooks'); // Performance targets and thresholds const PERFORMANCE_TARGETS = { totalExecutionTime: 30 * 60 * 1000, // 30 minutes individualPageLoad: 3000, // 3 seconds formSubmission: 5000, // 5 seconds authentication: 2000, // 2 seconds crossAgentWorkflow: 30000, // 30 seconds concurrentUserLimit: 5, networkRequestLimit: 100, // per page memoryThreshold: 512 * 1024 * 1024, // 512MB cpuThreshold: 80, // 80% CPU usage errorRate: 0.05 // 5% error rate threshold }; class PerformanceMonitor { constructor() { this.isMonitoring = false; this.metrics = { startTime: null, endTime: null, memoryBaseline: null, cpuBaseline: null, networkRequests: [], pageLoadTimes: [], agentPerformance: {}, concurrentSessions: [], errors: [], browserResources: {}, optimizationRecommendations: [] }; this.performanceObserver = null; this.browserPage = null; } /** * Initialize performance monitoring */ async initialize(browserPage = null) { this.browserPage = browserPage; this.metrics.startTime = Date.now(); this.metrics.memoryBaseline = process.memoryUsage(); // Set up performance observer for Node.js metrics this.setupPerformanceObserver(); // Set up browser performance monitoring if page is provided if (this.browserPage) { await this.setupBrowserMonitoring(); } this.isMonitoring = true; console.log('Performance Monitor initialized'); console.log(`Memory baseline: ${this.formatBytes(this.metrics.memoryBaseline.heapUsed)}`); } /** * Start monitoring a specific agent implementation */ async startAgentMonitoring(agentName) { if (!this.isMonitoring) { throw new Error('Performance monitor not initialized'); } const agentStartTime = Date.now(); this.metrics.agentPerformance[agentName] = { startTime: agentStartTime, endTime: null, duration: null, memoryUsage: process.memoryUsage(), pageLoads: [], networkRequests: [], errors: [], resourceUtilization: await this.getBrowserResourceUsage() }; console.log(`Started monitoring agent: ${agentName}`); return agentStartTime; } /** * Stop monitoring a specific agent implementation */ async stopAgentMonitoring(agentName) { if (!this.metrics.agentPerformance[agentName]) { console.warn(`Agent ${agentName} was not being monitored`); return null; } const agentEndTime = Date.now(); const agentData = this.metrics.agentPerformance[agentName]; agentData.endTime = agentEndTime; agentData.duration = agentEndTime - agentData.startTime; agentData.finalMemoryUsage = process.memoryUsage(); agentData.finalResourceUtilization = await this.getBrowserResourceUsage(); // Calculate agent-specific performance metrics agentData.performanceRating = this.calculateAgentPerformanceRating(agentData); console.log(`Stopped monitoring agent: ${agentName}`); console.log(` Duration: ${(agentData.duration / 1000).toFixed(1)}s`); console.log(` Performance Rating: ${agentData.performanceRating}`); return agentData; } /** * Record page load time */ recordPageLoad(url, loadTime, agentName = null) { const pageLoadData = { url, loadTime, timestamp: Date.now(), agent: agentName, exceededTarget: loadTime > PERFORMANCE_TARGETS.individualPageLoad }; this.metrics.pageLoadTimes.push(pageLoadData); // Add to agent-specific data if available if (agentName && this.metrics.agentPerformance[agentName]) { this.metrics.agentPerformance[agentName].pageLoads.push(pageLoadData); } if (pageLoadData.exceededTarget) { console.warn(`Page load exceeded target: ${url} (${loadTime}ms > ${PERFORMANCE_TARGETS.individualPageLoad}ms)`); } return pageLoadData; } /** * Record network request */ recordNetworkRequest(requestData, agentName = null) { const networkData = { ...requestData, timestamp: Date.now(), agent: agentName }; this.metrics.networkRequests.push(networkData); // Add to agent-specific data if available if (agentName && this.metrics.agentPerformance[agentName]) { this.metrics.agentPerformance[agentName].networkRequests.push(networkData); } return networkData; } /** * Record error */ recordError(error, context = '', agentName = null) { const errorData = { message: error.message || error, stack: error.stack || null, context, timestamp: Date.now(), agent: agentName }; this.metrics.errors.push(errorData); // Add to agent-specific data if available if (agentName && this.metrics.agentPerformance[agentName]) { this.metrics.agentPerformance[agentName].errors.push(errorData); } return errorData; } /** * Execute concurrent load testing */ async executeConcurrentLoadTest(sessionConfigs) { console.log(`Starting concurrent load test with ${sessionConfigs.length} sessions...`); const concurrentPromises = sessionConfigs.map((config, index) => this.simulateUserSession(`session_${index}`, config) ); const startTime = Date.now(); const results = await Promise.allSettled(concurrentPromises); const endTime = Date.now(); const loadTestResults = { totalSessions: sessionConfigs.length, successfulSessions: results.filter(r => r.status === 'fulfilled').length, failedSessions: results.filter(r => r.status === 'rejected').length, totalDuration: endTime - startTime, averageSessionDuration: 0, concurrentPerformanceRating: 'unknown', sessions: [] }; // Process individual session results results.forEach((result, index) => { const sessionId = `session_${index}`; if (result.status === 'fulfilled') { const sessionData = result.value; loadTestResults.sessions.push(sessionData); loadTestResults.averageSessionDuration += sessionData.duration; } else { loadTestResults.sessions.push({ sessionId, status: 'failed', error: result.reason.message || result.reason, duration: 0 }); this.recordError(result.reason, `Concurrent session ${sessionId}`); } }); if (loadTestResults.successfulSessions > 0) { loadTestResults.averageSessionDuration /= loadTestResults.successfulSessions; } // Calculate concurrent performance rating const successRate = loadTestResults.successfulSessions / loadTestResults.totalSessions; const avgDurationUnderTarget = loadTestResults.averageSessionDuration <= PERFORMANCE_TARGETS.crossAgentWorkflow; if (successRate >= 0.9 && avgDurationUnderTarget) { loadTestResults.concurrentPerformanceRating = 'EXCELLENT'; } else if (successRate >= 0.8 && avgDurationUnderTarget) { loadTestResults.concurrentPerformanceRating = 'GOOD'; } else if (successRate >= 0.6) { loadTestResults.concurrentPerformanceRating = 'ACCEPTABLE'; } else { loadTestResults.concurrentPerformanceRating = 'POOR'; } this.metrics.concurrentSessions.push(loadTestResults); console.log(`Concurrent load test completed:`); console.log(` Success rate: ${(successRate * 100).toFixed(1)}%`); console.log(` Average duration: ${(loadTestResults.averageSessionDuration / 1000).toFixed(1)}s`); console.log(` Performance rating: ${loadTestResults.concurrentPerformanceRating}`); return loadTestResults; } /** * Simulate individual user session */ async simulateUserSession(sessionId, config) { const sessionStartTime = Date.now(); const sessionData = { sessionId, startTime: sessionStartTime, endTime: null, duration: null, status: 'running', config, actions: [], errors: [] }; try { // Simulate user actions based on config for (const action of config.actions) { const actionStartTime = Date.now(); try { await this.simulateUserAction(action, sessionData); sessionData.actions.push({ action: action.type, duration: Date.now() - actionStartTime, status: 'completed' }); } catch (actionError) { sessionData.actions.push({ action: action.type, duration: Date.now() - actionStartTime, status: 'failed', error: actionError.message }); sessionData.errors.push(actionError); } } sessionData.status = 'completed'; } catch (sessionError) { sessionData.status = 'failed'; sessionData.errors.push(sessionError); } sessionData.endTime = Date.now(); sessionData.duration = sessionData.endTime - sessionData.startTime; return sessionData; } /** * Simulate individual user action */ async simulateUserAction(action, sessionData) { // Mock implementation of user actions switch (action.type) { case 'login': await this.wait(action.duration || 2000); this.recordPageLoad('/login', action.duration || 2000, sessionData.sessionId); break; case 'navigate': await this.wait(action.duration || 1500); this.recordPageLoad(action.url || '/dashboard', action.duration || 1500, sessionData.sessionId); break; case 'form_submit': await this.wait(action.duration || 3000); this.recordPageLoad(action.url || '/form', action.duration || 3000, sessionData.sessionId); break; case 'create_event': await this.wait(action.duration || 5000); this.recordPageLoad('/create-event', action.duration || 5000, sessionData.sessionId); break; default: await this.wait(1000); // Default action duration } } /** * Get current performance snapshot */ async getPerformanceSnapshot() { const currentTime = Date.now(); const currentMemory = process.memoryUsage(); const browserResources = await this.getBrowserResourceUsage(); return { timestamp: currentTime, elapsedTime: currentTime - this.metrics.startTime, memory: { current: currentMemory, baseline: this.metrics.memoryBaseline, delta: { heapUsed: currentMemory.heapUsed - this.metrics.memoryBaseline.heapUsed, heapTotal: currentMemory.heapTotal - this.metrics.memoryBaseline.heapTotal, external: currentMemory.external - this.metrics.memoryBaseline.external } }, browserResources, pageLoads: { total: this.metrics.pageLoadTimes.length, averageTime: this.calculateAveragePageLoadTime(), exceedingTarget: this.metrics.pageLoadTimes.filter(p => p.exceededTarget).length }, networkRequests: { total: this.metrics.networkRequests.length, averageResponseTime: this.calculateAverageNetworkResponseTime() }, errors: { total: this.metrics.errors.length, errorRate: this.calculateErrorRate() } }; } /** * Generate performance benchmarking report */ async generatePerformanceBenchmark() { const benchmark = { overall: { totalExecutionTime: Date.now() - this.metrics.startTime, targetExecutionTime: PERFORMANCE_TARGETS.totalExecutionTime, performanceMet: false, rating: 'UNKNOWN' }, agents: {}, pageLoads: { total: this.metrics.pageLoadTimes.length, averageTime: this.calculateAveragePageLoadTime(), target: PERFORMANCE_TARGETS.individualPageLoad, exceedingTarget: this.metrics.pageLoadTimes.filter(p => p.exceededTarget).length, rating: 'UNKNOWN' }, memory: { current: process.memoryUsage(), baseline: this.metrics.memoryBaseline, peakUsage: this.calculatePeakMemoryUsage(), threshold: PERFORMANCE_TARGETS.memoryThreshold, rating: 'UNKNOWN' }, concurrency: { maxConcurrentSessions: Math.max(...this.metrics.concurrentSessions.map(c => c.totalSessions), 0), averageSuccessRate: this.calculateAverageConcurrentSuccessRate(), target: PERFORMANCE_TARGETS.concurrentUserLimit, rating: 'UNKNOWN' }, recommendations: [] }; // Calculate overall performance rating benchmark.overall.performanceMet = benchmark.overall.totalExecutionTime <= benchmark.overall.targetExecutionTime; benchmark.overall.rating = benchmark.overall.performanceMet ? 'PASSED' : 'EXCEEDED'; // Calculate page load rating const avgPageLoadUnderTarget = benchmark.pageLoads.averageTime <= benchmark.pageLoads.target; const exceedingTargetRatio = benchmark.pageLoads.exceedingTarget / benchmark.pageLoads.total; if (avgPageLoadUnderTarget && exceedingTargetRatio <= 0.1) { benchmark.pageLoads.rating = 'EXCELLENT'; } else if (avgPageLoadUnderTarget && exceedingTargetRatio <= 0.2) { benchmark.pageLoads.rating = 'GOOD'; } else if (exceedingTargetRatio <= 0.4) { benchmark.pageLoads.rating = 'ACCEPTABLE'; } else { benchmark.pageLoads.rating = 'POOR'; } // Calculate memory rating const currentMemoryUsage = benchmark.memory.current.heapUsed; const memoryUnderThreshold = currentMemoryUsage <= benchmark.memory.threshold; benchmark.memory.rating = memoryUnderThreshold ? 'GOOD' : 'EXCEEDED'; // Calculate concurrency rating if (benchmark.concurrency.averageSuccessRate >= 0.9) { benchmark.concurrency.rating = 'EXCELLENT'; } else if (benchmark.concurrency.averageSuccessRate >= 0.8) { benchmark.concurrency.rating = 'GOOD'; } else if (benchmark.concurrency.averageSuccessRate >= 0.6) { benchmark.concurrency.rating = 'ACCEPTABLE'; } else { benchmark.concurrency.rating = 'POOR'; } // Calculate agent-specific benchmarks Object.keys(this.metrics.agentPerformance).forEach(agentName => { const agentData = this.metrics.agentPerformance[agentName]; benchmark.agents[agentName] = { duration: agentData.duration, performanceRating: agentData.performanceRating, pageLoads: agentData.pageLoads.length, averagePageLoadTime: this.calculateAveragePageLoadTime(agentData.pageLoads), errors: agentData.errors.length, memoryDelta: agentData.finalMemoryUsage ? agentData.finalMemoryUsage.heapUsed - agentData.memoryUsage.heapUsed : 0 }; }); // Generate optimization recommendations benchmark.recommendations = this.generateOptimizationRecommendations(benchmark); return benchmark; } /** * Generate optimization recommendations */ generateOptimizationRecommendations(benchmark) { const recommendations = []; // Overall execution time recommendations if (!benchmark.overall.performanceMet) { recommendations.push({ category: 'execution_time', priority: 'high', message: `Total execution time (${(benchmark.overall.totalExecutionTime / 60000).toFixed(1)}m) exceeds target (${(benchmark.overall.targetExecutionTime / 60000).toFixed(1)}m)`, suggestions: [ 'Implement more parallel test execution', 'Optimize page load waiting strategies', 'Reduce test data setup time', 'Consider test suite partitioning' ] }); } // Page load recommendations if (benchmark.pageLoads.rating === 'POOR' || benchmark.pageLoads.rating === 'ACCEPTABLE') { recommendations.push({ category: 'page_loads', priority: 'medium', message: `Average page load time (${benchmark.pageLoads.averageTime.toFixed(0)}ms) needs optimization`, suggestions: [ 'Implement page pre-loading strategies', 'Use more specific element waiting', 'Optimize CSS and JavaScript loading', 'Consider headless browser optimization' ] }); } // Memory recommendations if (benchmark.memory.rating === 'EXCEEDED') { recommendations.push({ category: 'memory', priority: 'medium', message: `Memory usage (${this.formatBytes(benchmark.memory.current.heapUsed)}) exceeds threshold`, suggestions: [ 'Implement browser instance cleanup', 'Reduce test data retention', 'Optimize page object lifecycle management', 'Consider garbage collection tuning' ] }); } // Concurrency recommendations if (benchmark.concurrency.rating === 'POOR' || benchmark.concurrency.rating === 'ACCEPTABLE') { recommendations.push({ category: 'concurrency', priority: 'medium', message: `Concurrent session success rate (${(benchmark.concurrency.averageSuccessRate * 100).toFixed(1)}%) needs improvement`, suggestions: [ 'Implement better resource isolation', 'Add retry mechanisms for failed sessions', 'Optimize database connection handling', 'Consider staggered session startup' ] }); } // Agent-specific recommendations Object.keys(benchmark.agents).forEach(agentName => { const agentData = benchmark.agents[agentName]; if (agentData.performanceRating === 'POOR') { recommendations.push({ category: 'agent_specific', priority: 'medium', message: `Agent ${agentName} performance needs optimization`, suggestions: [ `Optimize ${agentName} test execution flow`, 'Review page object efficiency', 'Consider agent-specific parallelization', 'Implement smart waiting strategies' ] }); } }); return recommendations; } /** * Export performance report */ async exportPerformanceReport(outputPath) { const benchmark = await this.generatePerformanceBenchmark(); const snapshot = await this.getPerformanceSnapshot(); const report = { metadata: { generatedAt: new Date().toISOString(), testDuration: Date.now() - this.metrics.startTime, environment: { nodeVersion: process.version, platform: process.platform, arch: process.arch } }, summary: { overallRating: benchmark.overall.rating, pageLoadRating: benchmark.pageLoads.rating, memoryRating: benchmark.memory.rating, concurrencyRating: benchmark.concurrency.rating, totalErrors: this.metrics.errors.length, totalPageLoads: this.metrics.pageLoadTimes.length, totalNetworkRequests: this.metrics.networkRequests.length }, benchmark, snapshot, rawMetrics: { agentPerformance: this.metrics.agentPerformance, pageLoadTimes: this.metrics.pageLoadTimes, networkRequests: this.metrics.networkRequests.length, // Don't include full request data errors: this.metrics.errors, concurrentSessions: this.metrics.concurrentSessions }, recommendations: benchmark.recommendations }; await fs.writeFile(outputPath, JSON.stringify(report, null, 2)); console.log(`Performance report exported to: ${outputPath}`); return report; } // ================= // Helper Methods // ================= /** * Setup performance observer for Node.js metrics */ setupPerformanceObserver() { this.performanceObserver = new PerformanceObserver((list) => { const entries = list.getEntries(); entries.forEach(entry => { if (entry.entryType === 'measure') { // Record custom performance measurements console.log(`Performance measurement: ${entry.name} - ${entry.duration.toFixed(2)}ms`); } }); }); this.performanceObserver.observe({ entryTypes: ['measure', 'navigation', 'resource'] }); } /** * Setup browser performance monitoring */ async setupBrowserMonitoring() { if (!this.browserPage) return; try { // Listen for network requests this.browserPage.on('request', (request) => { this.recordNetworkRequest({ url: request.url(), method: request.method(), resourceType: request.resourceType() }); }); // Listen for console errors this.browserPage.on('console', (msg) => { if (msg.type() === 'error') { this.recordError(new Error(msg.text()), 'Browser console error'); } }); // Listen for page errors this.browserPage.on('pageerror', (error) => { this.recordError(error, 'Page error'); }); console.log('Browser performance monitoring setup complete'); } catch (error) { console.warn('Failed to setup browser monitoring:', error.message); } } /** * Get browser resource usage */ async getBrowserResourceUsage() { if (!this.browserPage) { return { jsHeapSize: 0, totalJSHeapSize: 0, usedJSHeapSize: 0 }; } try { const metrics = await this.browserPage.evaluate(() => { if (performance.memory) { return { jsHeapSize: performance.memory.jsHeapSizeLimit, totalJSHeapSize: performance.memory.totalJSHeapSize, usedJSHeapSize: performance.memory.usedJSHeapSize }; } return { jsHeapSize: 0, totalJSHeapSize: 0, usedJSHeapSize: 0 }; }); return metrics; } catch (error) { console.warn('Failed to get browser resource usage:', error.message); return { jsHeapSize: 0, totalJSHeapSize: 0, usedJSHeapSize: 0 }; } } /** * Calculate agent performance rating */ calculateAgentPerformanceRating(agentData) { const duration = agentData.duration; const errorCount = agentData.errors.length; const pageLoadCount = agentData.pageLoads.length; const averagePageLoadTime = pageLoadCount > 0 ? agentData.pageLoads.reduce((sum, p) => sum + p.loadTime, 0) / pageLoadCount : 0; // Simple rating algorithm let score = 100; // Duration penalty (assuming 5 minutes is reasonable for an agent) if (duration > 300000) score -= 30; // 5+ minutes else if (duration > 180000) score -= 15; // 3-5 minutes else if (duration > 60000) score -= 5; // 1-3 minutes // Error penalty score -= errorCount * 10; // Page load penalty if (averagePageLoadTime > PERFORMANCE_TARGETS.individualPageLoad) { score -= 20; } // Memory penalty (if memory usage increased significantly) if (agentData.finalMemoryUsage && agentData.memoryUsage) { const memoryIncrease = agentData.finalMemoryUsage.heapUsed - agentData.memoryUsage.heapUsed; if (memoryIncrease > 50 * 1024 * 1024) { // 50MB increase score -= 10; } } // Convert score to rating if (score >= 90) return 'EXCELLENT'; if (score >= 80) return 'GOOD'; if (score >= 70) return 'ACCEPTABLE'; if (score >= 60) return 'POOR'; return 'CRITICAL'; } /** * Calculate average page load time */ calculateAveragePageLoadTime(pageLoads = null) { const loads = pageLoads || this.metrics.pageLoadTimes; if (loads.length === 0) return 0; return loads.reduce((sum, p) => sum + p.loadTime, 0) / loads.length; } /** * Calculate average network response time */ calculateAverageNetworkResponseTime() { const requestsWithDuration = this.metrics.networkRequests.filter(r => r.duration); if (requestsWithDuration.length === 0) return 0; return requestsWithDuration.reduce((sum, r) => sum + r.duration, 0) / requestsWithDuration.length; } /** * Calculate error rate */ calculateErrorRate() { const totalOperations = this.metrics.pageLoadTimes.length + this.metrics.networkRequests.length; if (totalOperations === 0) return 0; return this.metrics.errors.length / totalOperations; } /** * Calculate peak memory usage */ calculatePeakMemoryUsage() { // For now, return current memory as peak // In a real implementation, this would track peak usage over time return process.memoryUsage().heapUsed; } /** * Calculate average concurrent success rate */ calculateAverageConcurrentSuccessRate() { if (this.metrics.concurrentSessions.length === 0) return 1.0; const totalSuccessRate = this.metrics.concurrentSessions.reduce((sum, session) => { return sum + (session.successfulSessions / session.totalSessions); }, 0); return totalSuccessRate / this.metrics.concurrentSessions.length; } /** * Format bytes for human-readable output */ formatBytes(bytes) { if (bytes === 0) return '0 Bytes'; const k = 1024; const sizes = ['Bytes', 'KB', 'MB', 'GB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } /** * Wait utility */ async wait(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Cleanup performance monitoring */ async cleanup() { this.isMonitoring = false; this.metrics.endTime = Date.now(); if (this.performanceObserver) { this.performanceObserver.disconnect(); } console.log('Performance Monitor cleanup complete'); } } module.exports = PerformanceMonitor;