b3b0901cd6
- Added visual-regression.sh for detecting UI changes - Created optimize-tests.sh to improve test performance - Added canary-deploy.sh for safer deployments with automatic rollback - Enhanced overall testing and deployment reliability 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
390 lines
No EOL
12 KiB
Bash
Executable file
390 lines
No EOL
12 KiB
Bash
Executable file
#!/bin/bash
|
|
# visual-regression.sh - Script for visual regression testing
|
|
# Usage: ./bin/visual-regression.sh [capture|compare|report] [--threshold=5] [--page=login]
|
|
|
|
set -e
|
|
|
|
# Colors for output
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
RED='\033[0;31m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Default settings
|
|
ACTION=""
|
|
DIFF_THRESHOLD=5
|
|
PAGE_FILTER=""
|
|
CURRENT_DATE=$(date +"%Y-%m-%d")
|
|
BASELINE_DIR="tests/e2e/visual-regression/baseline"
|
|
CURRENT_DIR="tests/e2e/visual-regression/current"
|
|
DIFF_DIR="tests/e2e/visual-regression/diff"
|
|
REPORT_DIR="tests/e2e/visual-regression/reports"
|
|
|
|
# Parse arguments
|
|
for arg in "$@"; do
|
|
case $arg in
|
|
capture|compare|report)
|
|
ACTION="$arg"
|
|
shift
|
|
;;
|
|
--threshold=*)
|
|
DIFF_THRESHOLD="${arg#*=}"
|
|
shift
|
|
;;
|
|
--page=*)
|
|
PAGE_FILTER="${arg#*=}"
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Check if action is provided
|
|
if [ -z "$ACTION" ]; then
|
|
echo -e "${RED}Error: No action specified. Use: capture, compare, or report${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${GREEN}=== Visual Regression Testing - ${ACTION} ===${NC}"
|
|
|
|
# Check if we're in the right directory
|
|
if [ ! -d "tests/e2e" ]; then
|
|
echo -e "${RED}Error: Please run this script from the wordpress-dev directory${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if ImageMagick is installed
|
|
if ! command -v compare &> /dev/null; then
|
|
echo -e "${RED}Error: ImageMagick not found. Please install it to use this script.${NC}"
|
|
echo "On macOS: brew install imagemagick"
|
|
echo "On Ubuntu: sudo apt-get install imagemagick"
|
|
exit 1
|
|
fi
|
|
|
|
# Create directory structure
|
|
mkdir -p "$BASELINE_DIR" "$CURRENT_DIR" "$DIFF_DIR" "$REPORT_DIR"
|
|
|
|
# Function to capture screenshots for visual regression
|
|
capture_screenshots() {
|
|
echo -e "\n${YELLOW}Capturing screenshots for visual regression testing...${NC}"
|
|
|
|
# Create a temporary Playwright test file
|
|
TEMP_TEST="tests/e2e/visual-regression-temp.spec.ts"
|
|
|
|
echo "Creating test file: $TEMP_TEST"
|
|
cat > "$TEMP_TEST" << 'EOF'
|
|
import { test } from '@playwright/test';
|
|
import { LoginPage } from './pages/LoginPage';
|
|
import { DashboardPage } from './pages/DashboardPage';
|
|
import { TEST_USERS } from './data/test-users';
|
|
|
|
// Critical pages to capture
|
|
test('Capture critical pages for visual regression', async ({ page }) => {
|
|
// Get target directory from environment variable
|
|
const targetDir = process.env.SCREENSHOT_DIR || 'tests/e2e/visual-regression/current';
|
|
console.log(`Capturing screenshots to: ${targetDir}`);
|
|
|
|
// Login page
|
|
const loginPage = new LoginPage(page);
|
|
await loginPage.navigate();
|
|
await page.waitForLoadState('networkidle');
|
|
await page.screenshot({ path: `${targetDir}/login-page.png`, fullPage: true });
|
|
console.log('Captured login page');
|
|
|
|
// Login with test user
|
|
await loginPage.login(TEST_USERS.trainer.username, TEST_USERS.trainer.password);
|
|
|
|
// Dashboard page
|
|
const dashboardPage = new DashboardPage(page);
|
|
await dashboardPage.waitForDashboard();
|
|
await page.waitForLoadState('networkidle');
|
|
await page.screenshot({ path: `${targetDir}/dashboard-page.png`, fullPage: true });
|
|
console.log('Captured dashboard page');
|
|
|
|
// Events list
|
|
if (await dashboardPage.isEventsTableVisible()) {
|
|
await dashboardPage.filterEvents('all');
|
|
await page.waitForLoadState('networkidle');
|
|
await page.screenshot({ path: `${targetDir}/events-list.png`, fullPage: true });
|
|
console.log('Captured events list');
|
|
}
|
|
|
|
// Create event page
|
|
await dashboardPage.clickCreateEvent();
|
|
await page.waitForLoadState('networkidle');
|
|
await page.screenshot({ path: `${targetDir}/create-event-page.png`, fullPage: true });
|
|
console.log('Captured create event page');
|
|
|
|
// Certificate report page (if available)
|
|
try {
|
|
await page.goto('/certificates-report/');
|
|
await page.waitForLoadState('networkidle');
|
|
await page.screenshot({ path: `${targetDir}/certificate-report-page.png`, fullPage: true });
|
|
console.log('Captured certificate report page');
|
|
} catch (error) {
|
|
console.log('Certificate report page not available');
|
|
}
|
|
|
|
console.log('Screenshot capture complete');
|
|
});
|
|
EOF
|
|
|
|
# Run the test with the appropriate directory
|
|
echo -e "${YELLOW}Running screenshot capture...${NC}"
|
|
if [ -n "$PAGE_FILTER" ]; then
|
|
echo "Capturing only ${PAGE_FILTER} page"
|
|
SCREENSHOT_DIR="$CURRENT_DIR" npx playwright test "$TEMP_TEST" --grep="$PAGE_FILTER"
|
|
else
|
|
SCREENSHOT_DIR="$CURRENT_DIR" npx playwright test "$TEMP_TEST"
|
|
fi
|
|
|
|
# Clean up
|
|
rm "$TEMP_TEST"
|
|
|
|
echo -e "${GREEN}Screenshots captured successfully to ${CURRENT_DIR}${NC}"
|
|
}
|
|
|
|
# Function to establish baseline screenshots
|
|
establish_baseline() {
|
|
echo -e "\n${YELLOW}Establishing baseline screenshots...${NC}"
|
|
|
|
# Check if baseline directory is empty
|
|
if [ -z "$(ls -A "$BASELINE_DIR" 2>/dev/null)" ]; then
|
|
echo "Baseline directory is empty, capturing baseline screenshots..."
|
|
|
|
# Set SCREENSHOT_DIR to baseline for the capture script
|
|
SCREENSHOT_DIR="$BASELINE_DIR" capture_screenshots
|
|
else
|
|
echo "Baseline already exists. Use --force to overwrite."
|
|
|
|
# Optionally copy current screenshots to baseline
|
|
read -p "Copy current screenshots to baseline? (y/n) " -n 1 -r
|
|
echo
|
|
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
|
cp "$CURRENT_DIR"/*.png "$BASELINE_DIR"/ 2>/dev/null || true
|
|
echo "Current screenshots copied to baseline."
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Function to compare screenshots
|
|
compare_screenshots() {
|
|
echo -e "\n${YELLOW}Comparing screenshots against baseline...${NC}"
|
|
|
|
# Check if baseline exists
|
|
if [ -z "$(ls -A "$BASELINE_DIR" 2>/dev/null)" ]; then
|
|
echo -e "${RED}Error: No baseline screenshots found. Run with 'capture' action first.${NC}"
|
|
return 1
|
|
fi
|
|
|
|
# Check if current screenshots exist
|
|
if [ -z "$(ls -A "$CURRENT_DIR" 2>/dev/null)" ]; then
|
|
echo -e "${RED}Error: No current screenshots found. Run with 'capture' action first.${NC}"
|
|
return 1
|
|
fi
|
|
|
|
# Create comparison report file
|
|
REPORT_FILE="$REPORT_DIR/visual-diff-report-${CURRENT_DATE}.md"
|
|
echo "# Visual Regression Test Report - ${CURRENT_DATE}" > "$REPORT_FILE"
|
|
echo "" >> "$REPORT_FILE"
|
|
echo "| Page | Diff % | Status | Diff Image |" >> "$REPORT_FILE"
|
|
echo "|------|--------|--------|------------|" >> "$REPORT_FILE"
|
|
|
|
FAILURE_COUNT=0
|
|
SUCCESS_COUNT=0
|
|
|
|
# Loop through each baseline image
|
|
for baseline in "$BASELINE_DIR"/*.png; do
|
|
filename=$(basename "$baseline")
|
|
|
|
# If page filter is specified, only process that page
|
|
if [ -n "$PAGE_FILTER" ] && [[ ! "$filename" == *"$PAGE_FILTER"* ]]; then
|
|
continue
|
|
fi
|
|
|
|
current="$CURRENT_DIR/$filename"
|
|
diff="$DIFF_DIR/$filename"
|
|
|
|
# Skip if current image doesn't exist
|
|
if [ ! -f "$current" ]; then
|
|
echo -e "${YELLOW}Warning: Current screenshot ${filename} not found, skipping...${NC}"
|
|
continue
|
|
fi
|
|
|
|
# Compare images
|
|
echo -e "Comparing ${filename}..."
|
|
compare_output=$(compare -metric AE "$baseline" "$current" "$diff" 2>&1 || true)
|
|
|
|
# Get the pixel difference and calculate percentage
|
|
pixel_diff=$(echo "$compare_output" | grep -o '[0-9]\+' || echo "0")
|
|
total_pixels=$(identify -format "%w*%h" "$baseline" | bc)
|
|
diff_percentage=$(echo "scale=2; $pixel_diff * 100 / $total_pixels" | bc)
|
|
|
|
# Log the difference
|
|
if (( $(echo "$diff_percentage > $DIFF_THRESHOLD" | bc -l) )); then
|
|
echo -e "${RED}✗ ${filename} differs by ${diff_percentage}% (threshold: ${DIFF_THRESHOLD}%)${NC}"
|
|
status="❌ Failed"
|
|
FAILURE_COUNT=$((FAILURE_COUNT + 1))
|
|
else
|
|
echo -e "${GREEN}✓ ${filename} passed (${diff_percentage}% difference)${NC}"
|
|
status="✅ Passed"
|
|
SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
|
|
fi
|
|
|
|
# Add to report
|
|
echo "| ${filename} | ${diff_percentage}% | ${status} |  |" >> "$REPORT_FILE"
|
|
done
|
|
|
|
# Add summary to report
|
|
echo "" >> "$REPORT_FILE"
|
|
echo "## Summary" >> "$REPORT_FILE"
|
|
echo "" >> "$REPORT_FILE"
|
|
echo "- **Date:** ${CURRENT_DATE}" >> "$REPORT_FILE"
|
|
echo "- **Total Comparisons:** $((SUCCESS_COUNT + FAILURE_COUNT))" >> "$REPORT_FILE"
|
|
echo "- **Passed:** ${SUCCESS_COUNT}" >> "$REPORT_FILE"
|
|
echo "- **Failed:** ${FAILURE_COUNT}" >> "$REPORT_FILE"
|
|
echo "- **Threshold:** ${DIFF_THRESHOLD}%" >> "$REPORT_FILE"
|
|
|
|
# Output summary
|
|
echo -e "\n${GREEN}Comparison complete:${NC}"
|
|
echo "Total: $((SUCCESS_COUNT + FAILURE_COUNT))"
|
|
echo "Passed: ${SUCCESS_COUNT}"
|
|
echo "Failed: ${FAILURE_COUNT}"
|
|
echo "Report: ${REPORT_FILE}"
|
|
|
|
# Return success/failure
|
|
if [ $FAILURE_COUNT -eq 0 ]; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to generate a visual regression report
|
|
generate_report() {
|
|
echo -e "\n${YELLOW}Generating visual regression report...${NC}"
|
|
|
|
# Check if we have diff images
|
|
if [ -z "$(ls -A "$DIFF_DIR" 2>/dev/null)" ]; then
|
|
echo -e "${RED}Error: No diff images found. Run with 'compare' action first.${NC}"
|
|
return 1
|
|
fi
|
|
|
|
# Create an HTML report
|
|
HTML_REPORT="$REPORT_DIR/visual-diff-report-${CURRENT_DATE}.html"
|
|
|
|
cat > "$HTML_REPORT" << EOF
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Visual Regression Test Report - ${CURRENT_DATE}</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; margin: 20px; }
|
|
h1 { color: #333; }
|
|
.summary { margin: 20px 0; padding: 10px; background: #f5f5f5; border-radius: 5px; }
|
|
.comparison { margin-bottom: 30px; border: 1px solid #ddd; padding: 15px; border-radius: 5px; }
|
|
.comparison h3 { margin-top: 0; }
|
|
.comparison.fail { border-left: 5px solid #ff0000; }
|
|
.comparison.pass { border-left: 5px solid #00cc00; }
|
|
.images { display: flex; flex-wrap: wrap; }
|
|
.image-container { margin: 10px; text-align: center; }
|
|
img { max-width: 100%; border: 1px solid #ddd; }
|
|
.status { font-weight: bold; }
|
|
.pass-status { color: #00cc00; }
|
|
.fail-status { color: #ff0000; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Visual Regression Test Report - ${CURRENT_DATE}</h1>
|
|
|
|
<div class="summary">
|
|
<h2>Summary</h2>
|
|
<p><strong>Date:</strong> ${CURRENT_DATE}</p>
|
|
<p><strong>Threshold:</strong> ${DIFF_THRESHOLD}%</p>
|
|
EOF
|
|
|
|
# Add comparison sections
|
|
for diff in "$DIFF_DIR"/*.png; do
|
|
if [ -f "$diff" ]; then
|
|
filename=$(basename "$diff")
|
|
baseline="$BASELINE_DIR/$filename"
|
|
current="$CURRENT_DIR/$filename"
|
|
|
|
# Skip if baseline or current image doesn't exist
|
|
if [ ! -f "$baseline" ] || [ ! -f "$current" ]; then
|
|
continue
|
|
}
|
|
|
|
# Get the pixel difference and calculate percentage
|
|
compare_output=$(compare -metric AE "$baseline" "$current" "$diff" 2>&1 || true)
|
|
pixel_diff=$(echo "$compare_output" | grep -o '[0-9]\+' || echo "0")
|
|
total_pixels=$(identify -format "%w*%h" "$baseline" | bc)
|
|
diff_percentage=$(echo "scale=2; $pixel_diff * 100 / $total_pixels" | bc)
|
|
|
|
# Determine status
|
|
if (( $(echo "$diff_percentage > $DIFF_THRESHOLD" | bc -l) )); then
|
|
status_class="fail"
|
|
status_text="Failed"
|
|
status_style="fail-status"
|
|
else
|
|
status_class="pass"
|
|
status_text="Passed"
|
|
status_style="pass-status"
|
|
fi
|
|
|
|
# Add to HTML report
|
|
cat >> "$HTML_REPORT" << EOF
|
|
<div class="comparison ${status_class}">
|
|
<h3>${filename}</h3>
|
|
<p>Difference: ${diff_percentage}% - <span class="status ${status_style}">${status_text}</span></p>
|
|
<div class="images">
|
|
<div class="image-container">
|
|
<p>Baseline</p>
|
|
<img src="../baseline/${filename}" alt="Baseline">
|
|
</div>
|
|
<div class="image-container">
|
|
<p>Current</p>
|
|
<img src="../current/${filename}" alt="Current">
|
|
</div>
|
|
<div class="image-container">
|
|
<p>Diff</p>
|
|
<img src="../diff/${filename}" alt="Diff">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
EOF
|
|
fi
|
|
done
|
|
|
|
# Close HTML
|
|
cat >> "$HTML_REPORT" << EOF
|
|
</body>
|
|
</html>
|
|
EOF
|
|
|
|
echo -e "${GREEN}HTML report generated: ${HTML_REPORT}${NC}"
|
|
return 0
|
|
}
|
|
|
|
# Execute action
|
|
case $ACTION in
|
|
"capture")
|
|
capture_screenshots
|
|
;;
|
|
"baseline")
|
|
establish_baseline
|
|
;;
|
|
"compare")
|
|
capture_screenshots
|
|
compare_screenshots
|
|
;;
|
|
"report")
|
|
generate_report
|
|
;;
|
|
*)
|
|
echo -e "${RED}Invalid action: ${ACTION}${NC}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo -e "\n${GREEN}=== Visual Regression Testing Complete ===${NC}"
|
|
exit 0 |