feat: complete Phase 3.8 - test infrastructure and CI/CD enhancements
- Add test result artifacts storage with multiple formats (JUnit, JSON, HTML) - Configure GitHub Actions to upload and preserve test outputs - Add PR comment integration with test summaries - Create benchmark comparison workflow for PR performance tracking - Add detailed test report generation scripts - Configure artifact retention policies (30 days for tests, 90 for combined) - Set up test metadata collection for better debugging This completes all remaining test infrastructure tasks and provides comprehensive visibility into test results across CI/CD pipeline.
This commit is contained in:
154
.github/workflows/benchmark-pr.yml
vendored
Normal file
154
.github/workflows/benchmark-pr.yml
vendored
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
name: Benchmark PR Comparison
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- 'src/**'
|
||||||
|
- 'tests/benchmarks/**'
|
||||||
|
- 'package.json'
|
||||||
|
- 'vitest.config.benchmark.ts'
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
pull-requests: write
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
benchmark-comparison:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout PR branch
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: 'npm'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
# Run benchmarks on current branch
|
||||||
|
- name: Run current benchmarks
|
||||||
|
run: npm run benchmark:ci
|
||||||
|
|
||||||
|
- name: Save current results
|
||||||
|
run: cp benchmark-results.json benchmark-current.json
|
||||||
|
|
||||||
|
# Checkout and run benchmarks on base branch
|
||||||
|
- name: Checkout base branch
|
||||||
|
run: |
|
||||||
|
git checkout ${{ github.event.pull_request.base.sha }}
|
||||||
|
git status
|
||||||
|
|
||||||
|
- name: Install base dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Run baseline benchmarks
|
||||||
|
run: npm run benchmark:ci
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Save baseline results
|
||||||
|
run: |
|
||||||
|
if [ -f benchmark-results.json ]; then
|
||||||
|
cp benchmark-results.json benchmark-baseline.json
|
||||||
|
else
|
||||||
|
echo '{"files":[]}' > benchmark-baseline.json
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Compare results
|
||||||
|
- name: Checkout PR branch again
|
||||||
|
run: git checkout ${{ github.event.pull_request.head.sha }}
|
||||||
|
|
||||||
|
- name: Compare benchmarks
|
||||||
|
id: compare
|
||||||
|
run: |
|
||||||
|
node scripts/compare-benchmarks.js benchmark-current.json benchmark-baseline.json || echo "REGRESSION=true" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
# Upload comparison artifacts
|
||||||
|
- name: Upload benchmark comparison
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: benchmark-comparison-${{ github.run_number }}
|
||||||
|
path: |
|
||||||
|
benchmark-current.json
|
||||||
|
benchmark-baseline.json
|
||||||
|
benchmark-comparison.json
|
||||||
|
benchmark-comparison.md
|
||||||
|
retention-days: 30
|
||||||
|
|
||||||
|
# Post comparison to PR
|
||||||
|
- name: Post benchmark comparison to PR
|
||||||
|
if: always()
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const fs = require('fs');
|
||||||
|
let comment = '## ⚡ Benchmark Comparison\n\n';
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (fs.existsSync('benchmark-comparison.md')) {
|
||||||
|
const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
|
||||||
|
comment += comparison;
|
||||||
|
} else {
|
||||||
|
comment += 'Benchmark comparison could not be generated.';
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
comment += `Error reading benchmark comparison: ${error.message}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
comment += '\n\n---\n';
|
||||||
|
comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
|
||||||
|
|
||||||
|
// Find existing comment
|
||||||
|
const { data: comments } = await github.rest.issues.listComments({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: context.issue.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
const botComment = comments.find(comment =>
|
||||||
|
comment.user.type === 'Bot' &&
|
||||||
|
comment.body.includes('## ⚡ Benchmark Comparison')
|
||||||
|
);
|
||||||
|
|
||||||
|
if (botComment) {
|
||||||
|
await github.rest.issues.updateComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
comment_id: botComment.id,
|
||||||
|
body: comment
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await github.rest.issues.createComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: context.issue.number,
|
||||||
|
body: comment
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
# Add status check
|
||||||
|
- name: Set benchmark status
|
||||||
|
if: always()
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
|
||||||
|
const state = hasRegression ? 'failure' : 'success';
|
||||||
|
const description = hasRegression
|
||||||
|
? 'Performance regressions detected'
|
||||||
|
: 'No performance regressions';
|
||||||
|
|
||||||
|
await github.rest.repos.createCommitStatus({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
sha: context.sha,
|
||||||
|
state: state,
|
||||||
|
target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
|
||||||
|
description: description,
|
||||||
|
context: 'benchmarks/regression-check'
|
||||||
|
});
|
||||||
274
.github/workflows/test.yml
vendored
274
.github/workflows/test.yml
vendored
@@ -10,25 +10,281 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 20
|
node-version: 20
|
||||||
cache: 'npm'
|
cache: 'npm'
|
||||||
- run: npm ci
|
|
||||||
- run: npm run test:coverage # Run tests with coverage
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
# Run tests with coverage and multiple reporters
|
||||||
|
- name: Run tests with coverage
|
||||||
|
run: npm run test:coverage
|
||||||
|
env:
|
||||||
|
CI: true
|
||||||
|
|
||||||
|
# Generate test summary
|
||||||
|
- name: Generate test summary
|
||||||
|
if: always()
|
||||||
|
run: node scripts/generate-test-summary.js
|
||||||
|
|
||||||
|
# Generate detailed reports
|
||||||
|
- name: Generate detailed reports
|
||||||
|
if: always()
|
||||||
|
run: node scripts/generate-detailed-reports.js
|
||||||
|
|
||||||
|
# Upload test results artifacts
|
||||||
|
- name: Upload test results
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-results-${{ github.run_number }}-${{ github.run_attempt }}
|
||||||
|
path: |
|
||||||
|
test-results/
|
||||||
|
test-summary.md
|
||||||
|
test-reports/
|
||||||
|
retention-days: 30
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
# Upload coverage artifacts
|
||||||
|
- name: Upload coverage reports
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-${{ github.run_number }}-${{ github.run_attempt }}
|
||||||
|
path: |
|
||||||
|
coverage/
|
||||||
|
retention-days: 30
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
# Upload coverage to Codecov
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
|
if: always()
|
||||||
uses: codecov/codecov-action@v4
|
uses: codecov/codecov-action@v4
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.CODECOV_TOKEN }}
|
token: ${{ secrets.CODECOV_TOKEN }}
|
||||||
files: ./coverage/lcov.info
|
files: ./coverage/lcov.info
|
||||||
flags: unittests
|
flags: unittests
|
||||||
name: codecov-umbrella
|
name: codecov-umbrella
|
||||||
fail_ci_if_error: true
|
fail_ci_if_error: false
|
||||||
verbose: true
|
verbose: true
|
||||||
- run: npm run lint
|
|
||||||
- run: npm run typecheck || true # Allow to fail initially
|
|
||||||
|
|
||||||
# Run benchmarks as part of CI (without performance regression checks)
|
# Run linting
|
||||||
- name: Run benchmarks (smoke test)
|
- name: Run linting
|
||||||
run: npm run benchmark -- --run tests/benchmarks/sample.bench.ts
|
run: npm run lint
|
||||||
continue-on-error: true
|
|
||||||
|
# Run type checking
|
||||||
|
- name: Run type checking
|
||||||
|
run: npm run typecheck || true # Allow to fail initially
|
||||||
|
|
||||||
|
# Run benchmarks
|
||||||
|
- name: Run benchmarks
|
||||||
|
id: benchmarks
|
||||||
|
run: npm run benchmark:ci
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
# Upload benchmark results
|
||||||
|
- name: Upload benchmark results
|
||||||
|
if: always() && steps.benchmarks.outcome != 'skipped'
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: benchmark-results-${{ github.run_number }}-${{ github.run_attempt }}
|
||||||
|
path: |
|
||||||
|
benchmark-results.json
|
||||||
|
retention-days: 30
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
# Create test report comment for PRs
|
||||||
|
- name: Create test report comment
|
||||||
|
if: github.event_name == 'pull_request' && always()
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const fs = require('fs');
|
||||||
|
let summary = '## Test Results\n\nTest summary generation failed.';
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (fs.existsSync('test-summary.md')) {
|
||||||
|
summary = fs.readFileSync('test-summary.md', 'utf8');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error reading test summary:', error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find existing comment
|
||||||
|
const { data: comments } = await github.rest.issues.listComments({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: context.issue.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
const botComment = comments.find(comment =>
|
||||||
|
comment.user.type === 'Bot' &&
|
||||||
|
comment.body.includes('## Test Results')
|
||||||
|
);
|
||||||
|
|
||||||
|
if (botComment) {
|
||||||
|
// Update existing comment
|
||||||
|
await github.rest.issues.updateComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
comment_id: botComment.id,
|
||||||
|
body: summary
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Create new comment
|
||||||
|
await github.rest.issues.createComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: context.issue.number,
|
||||||
|
body: summary
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
# Generate job summary
|
||||||
|
- name: Generate job summary
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
echo "# Test Run Summary" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
if [ -f test-summary.md ]; then
|
||||||
|
cat test-summary.md >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "Test summary generation failed." >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "## 📥 Download Artifacts" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- [Test Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- [Coverage Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- [Benchmark Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
# Store test metadata
|
||||||
|
- name: Store test metadata
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
cat > test-metadata.json << EOF
|
||||||
|
{
|
||||||
|
"run_id": "${{ github.run_id }}",
|
||||||
|
"run_number": "${{ github.run_number }}",
|
||||||
|
"run_attempt": "${{ github.run_attempt }}",
|
||||||
|
"sha": "${{ github.sha }}",
|
||||||
|
"ref": "${{ github.ref }}",
|
||||||
|
"event_name": "${{ github.event_name }}",
|
||||||
|
"repository": "${{ github.repository }}",
|
||||||
|
"actor": "${{ github.actor }}",
|
||||||
|
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||||
|
"node_version": "$(node --version)",
|
||||||
|
"npm_version": "$(npm --version)"
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Upload test metadata
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-metadata-${{ github.run_number }}-${{ github.run_attempt }}
|
||||||
|
path: test-metadata.json
|
||||||
|
retention-days: 30
|
||||||
|
|
||||||
|
# Separate job to process and publish test results
|
||||||
|
publish-results:
|
||||||
|
needs: test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
permissions:
|
||||||
|
checks: write
|
||||||
|
pull-requests: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
# Download all artifacts
|
||||||
|
- name: Download all artifacts
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
path: artifacts
|
||||||
|
|
||||||
|
# Publish test results as checks
|
||||||
|
- name: Publish test results
|
||||||
|
uses: dorny/test-reporter@v1
|
||||||
|
if: always()
|
||||||
|
with:
|
||||||
|
name: Test Results
|
||||||
|
path: 'artifacts/test-results-*/test-results/junit.xml'
|
||||||
|
reporter: java-junit
|
||||||
|
fail-on-error: false
|
||||||
|
|
||||||
|
# Create a combined artifact with all results
|
||||||
|
- name: Create combined results artifact
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
mkdir -p combined-results
|
||||||
|
cp -r artifacts/* combined-results/ 2>/dev/null || true
|
||||||
|
|
||||||
|
# Create index file
|
||||||
|
cat > combined-results/index.html << 'EOF'
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>n8n-mcp Test Results</title>
|
||||||
|
<style>
|
||||||
|
body { font-family: Arial, sans-serif; margin: 40px; }
|
||||||
|
h1 { color: #333; }
|
||||||
|
.section { margin: 20px 0; padding: 20px; border: 1px solid #ddd; border-radius: 5px; }
|
||||||
|
a { color: #0066cc; text-decoration: none; }
|
||||||
|
a:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>n8n-mcp Test Results</h1>
|
||||||
|
<div class="section">
|
||||||
|
<h2>Test Reports</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-reports/report.html">📊 Detailed HTML Report</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-results/html/index.html">📈 Vitest HTML Report</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-reports/report.md">📄 Markdown Report</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-summary.md">📝 PR Summary</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-results/junit.xml">🔧 JUnit XML</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-results/results.json">🔢 JSON Results</a></li>
|
||||||
|
<li><a href="test-results-${{ github.run_number }}-${{ github.run_attempt }}/test-reports/report.json">📊 Full JSON Report</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="section">
|
||||||
|
<h2>Coverage Reports</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="coverage-${{ github.run_number }}-${{ github.run_attempt }}/html/index.html">HTML Coverage Report</a></li>
|
||||||
|
<li><a href="coverage-${{ github.run_number }}-${{ github.run_attempt }}/lcov.info">LCOV Report</a></li>
|
||||||
|
<li><a href="coverage-${{ github.run_number }}-${{ github.run_attempt }}/coverage-summary.json">Coverage Summary JSON</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="section">
|
||||||
|
<h2>Benchmark Results</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="benchmark-results-${{ github.run_number }}-${{ github.run_attempt }}/benchmark-results.json">Benchmark Results JSON</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="section">
|
||||||
|
<h2>Metadata</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="test-metadata-${{ github.run_number }}-${{ github.run_attempt }}/test-metadata.json">Test Run Metadata</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="section">
|
||||||
|
<p><em>Generated at $(date -u +%Y-%m-%dT%H:%M:%SZ)</em></p>
|
||||||
|
<p><em>Run: #${{ github.run_number }} | SHA: ${{ github.sha }}</em></p>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Upload combined results
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: all-test-results-${{ github.run_number }}
|
||||||
|
path: combined-results/
|
||||||
|
retention-days: 90
|
||||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -40,6 +40,14 @@ logs/
|
|||||||
coverage/
|
coverage/
|
||||||
.nyc_output/
|
.nyc_output/
|
||||||
test-results/
|
test-results/
|
||||||
|
test-reports/
|
||||||
|
test-summary.md
|
||||||
|
test-metadata.json
|
||||||
|
benchmark-results.json
|
||||||
|
benchmark-comparison.md
|
||||||
|
benchmark-comparison.json
|
||||||
|
benchmark-current.json
|
||||||
|
benchmark-baseline.json
|
||||||
tests/data/*.db
|
tests/data/*.db
|
||||||
tests/fixtures/*.tmp
|
tests/fixtures/*.tmp
|
||||||
.vitest/
|
.vitest/
|
||||||
|
|||||||
146
docs/test-artifacts.md
Normal file
146
docs/test-artifacts.md
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
# Test Artifacts Documentation
|
||||||
|
|
||||||
|
This document describes the comprehensive test result artifact storage system implemented in the n8n-mcp project.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The test artifact system captures, stores, and presents test results in multiple formats to facilitate debugging, analysis, and historical tracking of test performance.
|
||||||
|
|
||||||
|
## Artifact Types
|
||||||
|
|
||||||
|
### 1. Test Results
|
||||||
|
- **JUnit XML** (`test-results/junit.xml`): Standard format for CI integration
|
||||||
|
- **JSON Results** (`test-results/results.json`): Detailed test data for analysis
|
||||||
|
- **HTML Report** (`test-results/html/index.html`): Interactive test report
|
||||||
|
- **Test Summary** (`test-summary.md`): Markdown summary for PR comments
|
||||||
|
|
||||||
|
### 2. Coverage Reports
|
||||||
|
- **LCOV** (`coverage/lcov.info`): Standard coverage format
|
||||||
|
- **HTML Coverage** (`coverage/html/index.html`): Interactive coverage browser
|
||||||
|
- **Coverage Summary** (`coverage/coverage-summary.json`): JSON coverage data
|
||||||
|
|
||||||
|
### 3. Benchmark Results
|
||||||
|
- **Benchmark JSON** (`benchmark-results.json`): Raw benchmark data
|
||||||
|
- **Comparison Reports** (`benchmark-comparison.md`): PR benchmark comparisons
|
||||||
|
|
||||||
|
### 4. Detailed Reports
|
||||||
|
- **HTML Report** (`test-reports/report.html`): Comprehensive styled report
|
||||||
|
- **Markdown Report** (`test-reports/report.md`): Full markdown report
|
||||||
|
- **JSON Report** (`test-reports/report.json`): Complete test data
|
||||||
|
|
||||||
|
## GitHub Actions Integration
|
||||||
|
|
||||||
|
### Test Workflow (`test.yml`)
|
||||||
|
|
||||||
|
The main test workflow:
|
||||||
|
1. Runs tests with coverage using multiple reporters
|
||||||
|
2. Generates test summaries and detailed reports
|
||||||
|
3. Uploads artifacts with metadata
|
||||||
|
4. Posts summaries to PRs
|
||||||
|
5. Creates a combined artifact index
|
||||||
|
|
||||||
|
### Benchmark PR Workflow (`benchmark-pr.yml`)
|
||||||
|
|
||||||
|
For pull requests:
|
||||||
|
1. Runs benchmarks on PR branch
|
||||||
|
2. Runs benchmarks on base branch
|
||||||
|
3. Compares results
|
||||||
|
4. Posts comparison to PR
|
||||||
|
5. Sets status checks for regressions
|
||||||
|
|
||||||
|
## Artifact Retention
|
||||||
|
|
||||||
|
- **Test Results**: 30 days
|
||||||
|
- **Coverage Reports**: 30 days
|
||||||
|
- **Benchmark Results**: 30 days
|
||||||
|
- **Combined Results**: 90 days
|
||||||
|
- **Test Metadata**: 30 days
|
||||||
|
|
||||||
|
## PR Comment Integration
|
||||||
|
|
||||||
|
The system automatically:
|
||||||
|
- Posts test summaries to PR comments
|
||||||
|
- Updates existing comments instead of creating duplicates
|
||||||
|
- Includes links to full artifacts
|
||||||
|
- Shows coverage and benchmark changes
|
||||||
|
|
||||||
|
## Job Summary
|
||||||
|
|
||||||
|
Each workflow run includes a job summary with:
|
||||||
|
- Test results overview
|
||||||
|
- Coverage summary
|
||||||
|
- Benchmark results
|
||||||
|
- Direct links to download artifacts
|
||||||
|
|
||||||
|
## Local Development
|
||||||
|
|
||||||
|
### Running Tests with Reports
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run tests with all reporters
|
||||||
|
CI=true npm run test:coverage
|
||||||
|
|
||||||
|
# Generate detailed reports
|
||||||
|
node scripts/generate-detailed-reports.js
|
||||||
|
|
||||||
|
# Generate test summary
|
||||||
|
node scripts/generate-test-summary.js
|
||||||
|
|
||||||
|
# Compare benchmarks
|
||||||
|
node scripts/compare-benchmarks.js benchmark-results.json benchmark-baseline.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Report Locations
|
||||||
|
|
||||||
|
When running locally, reports are generated in:
|
||||||
|
- `test-results/` - Vitest outputs
|
||||||
|
- `test-reports/` - Detailed reports
|
||||||
|
- `coverage/` - Coverage reports
|
||||||
|
- Root directory - Summary files
|
||||||
|
|
||||||
|
## Report Formats
|
||||||
|
|
||||||
|
### HTML Report Features
|
||||||
|
- Responsive design
|
||||||
|
- Test suite breakdown
|
||||||
|
- Failed test details with error messages
|
||||||
|
- Coverage visualization with progress bars
|
||||||
|
- Benchmark performance metrics
|
||||||
|
- Sortable tables
|
||||||
|
|
||||||
|
### Markdown Report Features
|
||||||
|
- GitHub-compatible formatting
|
||||||
|
- Summary statistics
|
||||||
|
- Failed test listings
|
||||||
|
- Coverage breakdown
|
||||||
|
- Benchmark comparisons
|
||||||
|
|
||||||
|
### JSON Report Features
|
||||||
|
- Complete test data
|
||||||
|
- Programmatic access
|
||||||
|
- Historical comparison
|
||||||
|
- CI/CD integration
|
||||||
|
|
||||||
|
## Best Practices
|
||||||
|
|
||||||
|
1. **Always Check Artifacts**: When tests fail in CI, download and review the HTML report
|
||||||
|
2. **Monitor Coverage**: Use the coverage reports to identify untested code
|
||||||
|
3. **Track Benchmarks**: Review benchmark comparisons on performance-critical PRs
|
||||||
|
4. **Archive Important Runs**: Download artifacts from significant releases
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Missing Artifacts
|
||||||
|
- Check if tests ran to completion
|
||||||
|
- Verify artifact upload steps executed
|
||||||
|
- Check retention period hasn't expired
|
||||||
|
|
||||||
|
### Report Generation Failures
|
||||||
|
- Ensure all dependencies are installed
|
||||||
|
- Check for valid test/coverage output files
|
||||||
|
- Review workflow logs for errors
|
||||||
|
|
||||||
|
### PR Comment Issues
|
||||||
|
- Verify GitHub Actions permissions
|
||||||
|
- Check bot authentication
|
||||||
|
- Review comment posting logs
|
||||||
260
scripts/compare-benchmarks.js
Normal file
260
scripts/compare-benchmarks.js
Normal file
@@ -0,0 +1,260 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { readFileSync, existsSync, writeFileSync } from 'fs';
|
||||||
|
import { resolve } from 'path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare benchmark results between runs
|
||||||
|
*/
|
||||||
|
class BenchmarkComparator {
|
||||||
|
constructor() {
|
||||||
|
this.threshold = 0.1; // 10% threshold for significant changes
|
||||||
|
}
|
||||||
|
|
||||||
|
loadBenchmarkResults(path) {
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return JSON.parse(readFileSync(path, 'utf-8'));
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error loading benchmark results from ${path}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
compareBenchmarks(current, baseline) {
|
||||||
|
const comparison = {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
summary: {
|
||||||
|
improved: 0,
|
||||||
|
regressed: 0,
|
||||||
|
unchanged: 0,
|
||||||
|
added: 0,
|
||||||
|
removed: 0
|
||||||
|
},
|
||||||
|
benchmarks: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Create maps for easy lookup
|
||||||
|
const currentMap = new Map();
|
||||||
|
const baselineMap = new Map();
|
||||||
|
|
||||||
|
// Process current benchmarks
|
||||||
|
if (current && current.files) {
|
||||||
|
for (const file of current.files) {
|
||||||
|
for (const group of file.groups || []) {
|
||||||
|
for (const bench of group.benchmarks || []) {
|
||||||
|
const key = `${group.name}::${bench.name}`;
|
||||||
|
currentMap.set(key, {
|
||||||
|
ops: bench.result.hz,
|
||||||
|
mean: bench.result.mean,
|
||||||
|
file: file.filepath
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process baseline benchmarks
|
||||||
|
if (baseline && baseline.files) {
|
||||||
|
for (const file of baseline.files) {
|
||||||
|
for (const group of file.groups || []) {
|
||||||
|
for (const bench of group.benchmarks || []) {
|
||||||
|
const key = `${group.name}::${bench.name}`;
|
||||||
|
baselineMap.set(key, {
|
||||||
|
ops: bench.result.hz,
|
||||||
|
mean: bench.result.mean,
|
||||||
|
file: file.filepath
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare benchmarks
|
||||||
|
for (const [key, current] of currentMap) {
|
||||||
|
const baseline = baselineMap.get(key);
|
||||||
|
|
||||||
|
if (!baseline) {
|
||||||
|
// New benchmark
|
||||||
|
comparison.summary.added++;
|
||||||
|
comparison.benchmarks.push({
|
||||||
|
name: key,
|
||||||
|
status: 'added',
|
||||||
|
current: current.ops,
|
||||||
|
baseline: null,
|
||||||
|
change: null,
|
||||||
|
file: current.file
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Compare performance
|
||||||
|
const change = ((current.ops - baseline.ops) / baseline.ops) * 100;
|
||||||
|
let status = 'unchanged';
|
||||||
|
|
||||||
|
if (Math.abs(change) >= this.threshold * 100) {
|
||||||
|
if (change > 0) {
|
||||||
|
status = 'improved';
|
||||||
|
comparison.summary.improved++;
|
||||||
|
} else {
|
||||||
|
status = 'regressed';
|
||||||
|
comparison.summary.regressed++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
comparison.summary.unchanged++;
|
||||||
|
}
|
||||||
|
|
||||||
|
comparison.benchmarks.push({
|
||||||
|
name: key,
|
||||||
|
status,
|
||||||
|
current: current.ops,
|
||||||
|
baseline: baseline.ops,
|
||||||
|
change,
|
||||||
|
meanCurrent: current.mean,
|
||||||
|
meanBaseline: baseline.mean,
|
||||||
|
file: current.file
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for removed benchmarks
|
||||||
|
for (const [key, baseline] of baselineMap) {
|
||||||
|
if (!currentMap.has(key)) {
|
||||||
|
comparison.summary.removed++;
|
||||||
|
comparison.benchmarks.push({
|
||||||
|
name: key,
|
||||||
|
status: 'removed',
|
||||||
|
current: null,
|
||||||
|
baseline: baseline.ops,
|
||||||
|
change: null,
|
||||||
|
file: baseline.file
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by change percentage (regressions first)
|
||||||
|
comparison.benchmarks.sort((a, b) => {
|
||||||
|
if (a.status === 'regressed' && b.status !== 'regressed') return -1;
|
||||||
|
if (b.status === 'regressed' && a.status !== 'regressed') return 1;
|
||||||
|
if (a.change !== null && b.change !== null) {
|
||||||
|
return a.change - b.change;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
return comparison;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateMarkdownReport(comparison) {
|
||||||
|
let report = '## Benchmark Comparison Report\n\n';
|
||||||
|
|
||||||
|
const { summary } = comparison;
|
||||||
|
report += '### Summary\n\n';
|
||||||
|
report += `- **Improved**: ${summary.improved} benchmarks\n`;
|
||||||
|
report += `- **Regressed**: ${summary.regressed} benchmarks\n`;
|
||||||
|
report += `- **Unchanged**: ${summary.unchanged} benchmarks\n`;
|
||||||
|
report += `- **Added**: ${summary.added} benchmarks\n`;
|
||||||
|
report += `- **Removed**: ${summary.removed} benchmarks\n\n`;
|
||||||
|
|
||||||
|
// Regressions
|
||||||
|
const regressions = comparison.benchmarks.filter(b => b.status === 'regressed');
|
||||||
|
if (regressions.length > 0) {
|
||||||
|
report += '### ⚠️ Performance Regressions\n\n';
|
||||||
|
report += '| Benchmark | Current | Baseline | Change |\n';
|
||||||
|
report += '|-----------|---------|----------|--------|\n';
|
||||||
|
|
||||||
|
for (const bench of regressions) {
|
||||||
|
const currentOps = bench.current.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const baselineOps = bench.baseline.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const changeStr = bench.change.toFixed(2);
|
||||||
|
report += `| ${bench.name} | ${currentOps} ops/s | ${baselineOps} ops/s | **${changeStr}%** |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Improvements
|
||||||
|
const improvements = comparison.benchmarks.filter(b => b.status === 'improved');
|
||||||
|
if (improvements.length > 0) {
|
||||||
|
report += '### ✅ Performance Improvements\n\n';
|
||||||
|
report += '| Benchmark | Current | Baseline | Change |\n';
|
||||||
|
report += '|-----------|---------|----------|--------|\n';
|
||||||
|
|
||||||
|
for (const bench of improvements) {
|
||||||
|
const currentOps = bench.current.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const baselineOps = bench.baseline.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const changeStr = bench.change.toFixed(2);
|
||||||
|
report += `| ${bench.name} | ${currentOps} ops/s | ${baselineOps} ops/s | **+${changeStr}%** |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
// New benchmarks
|
||||||
|
const added = comparison.benchmarks.filter(b => b.status === 'added');
|
||||||
|
if (added.length > 0) {
|
||||||
|
report += '### 🆕 New Benchmarks\n\n';
|
||||||
|
report += '| Benchmark | Performance |\n';
|
||||||
|
report += '|-----------|-------------|\n';
|
||||||
|
|
||||||
|
for (const bench of added) {
|
||||||
|
const ops = bench.current.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
report += `| ${bench.name} | ${ops} ops/s |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateJsonReport(comparison) {
|
||||||
|
return JSON.stringify(comparison, null, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
async compare(currentPath, baselinePath) {
|
||||||
|
// Load results
|
||||||
|
const current = this.loadBenchmarkResults(currentPath);
|
||||||
|
const baseline = this.loadBenchmarkResults(baselinePath);
|
||||||
|
|
||||||
|
if (!current && !baseline) {
|
||||||
|
console.error('No benchmark results found');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate comparison
|
||||||
|
const comparison = this.compareBenchmarks(current, baseline);
|
||||||
|
|
||||||
|
// Generate reports
|
||||||
|
const markdownReport = this.generateMarkdownReport(comparison);
|
||||||
|
const jsonReport = this.generateJsonReport(comparison);
|
||||||
|
|
||||||
|
// Write reports
|
||||||
|
writeFileSync('benchmark-comparison.md', markdownReport);
|
||||||
|
writeFileSync('benchmark-comparison.json', jsonReport);
|
||||||
|
|
||||||
|
// Output summary to console
|
||||||
|
console.log(markdownReport);
|
||||||
|
|
||||||
|
// Return exit code based on regressions
|
||||||
|
if (comparison.summary.regressed > 0) {
|
||||||
|
console.error(`\n❌ Found ${comparison.summary.regressed} performance regressions`);
|
||||||
|
process.exit(1);
|
||||||
|
} else {
|
||||||
|
console.log(`\n✅ No performance regressions found`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse command line arguments
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
if (args.length < 1) {
|
||||||
|
console.error('Usage: node compare-benchmarks.js <current-results> [baseline-results]');
|
||||||
|
console.error('If baseline-results is not provided, it will look for benchmark-baseline.json');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentPath = args[0];
|
||||||
|
const baselinePath = args[1] || 'benchmark-baseline.json';
|
||||||
|
|
||||||
|
// Run comparison
|
||||||
|
const comparator = new BenchmarkComparator();
|
||||||
|
comparator.compare(currentPath, baselinePath).catch(console.error);
|
||||||
675
scripts/generate-detailed-reports.js
Normal file
675
scripts/generate-detailed-reports.js
Normal file
@@ -0,0 +1,675 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
|
||||||
|
import { resolve, dirname } from 'path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate detailed test reports in multiple formats
|
||||||
|
*/
|
||||||
|
class TestReportGenerator {
|
||||||
|
constructor() {
|
||||||
|
this.results = {
|
||||||
|
tests: null,
|
||||||
|
coverage: null,
|
||||||
|
benchmarks: null,
|
||||||
|
metadata: {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
repository: process.env.GITHUB_REPOSITORY || 'n8n-mcp',
|
||||||
|
sha: process.env.GITHUB_SHA || 'unknown',
|
||||||
|
branch: process.env.GITHUB_REF || 'unknown',
|
||||||
|
runId: process.env.GITHUB_RUN_ID || 'local',
|
||||||
|
runNumber: process.env.GITHUB_RUN_NUMBER || '0',
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
loadTestResults() {
|
||||||
|
const testResultPath = resolve(process.cwd(), 'test-results/results.json');
|
||||||
|
if (existsSync(testResultPath)) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(readFileSync(testResultPath, 'utf-8'));
|
||||||
|
this.results.tests = this.processTestResults(data);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading test results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processTestResults(data) {
|
||||||
|
const processedResults = {
|
||||||
|
summary: {
|
||||||
|
total: data.numTotalTests || 0,
|
||||||
|
passed: data.numPassedTests || 0,
|
||||||
|
failed: data.numFailedTests || 0,
|
||||||
|
skipped: data.numSkippedTests || 0,
|
||||||
|
duration: data.duration || 0,
|
||||||
|
success: (data.numFailedTests || 0) === 0
|
||||||
|
},
|
||||||
|
testSuites: [],
|
||||||
|
failedTests: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Process test suites
|
||||||
|
if (data.testResults) {
|
||||||
|
for (const suite of data.testResults) {
|
||||||
|
const suiteInfo = {
|
||||||
|
name: suite.name,
|
||||||
|
duration: suite.duration || 0,
|
||||||
|
tests: {
|
||||||
|
total: suite.numPassingTests + suite.numFailingTests + suite.numPendingTests,
|
||||||
|
passed: suite.numPassingTests || 0,
|
||||||
|
failed: suite.numFailingTests || 0,
|
||||||
|
skipped: suite.numPendingTests || 0
|
||||||
|
},
|
||||||
|
status: suite.numFailingTests === 0 ? 'passed' : 'failed'
|
||||||
|
};
|
||||||
|
|
||||||
|
processedResults.testSuites.push(suiteInfo);
|
||||||
|
|
||||||
|
// Collect failed tests
|
||||||
|
if (suite.testResults) {
|
||||||
|
for (const test of suite.testResults) {
|
||||||
|
if (test.status === 'failed') {
|
||||||
|
processedResults.failedTests.push({
|
||||||
|
suite: suite.name,
|
||||||
|
test: test.title,
|
||||||
|
duration: test.duration || 0,
|
||||||
|
error: test.failureMessages ? test.failureMessages.join('\n') : 'Unknown error'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedResults;
|
||||||
|
}
|
||||||
|
|
||||||
|
loadCoverageResults() {
|
||||||
|
const coveragePath = resolve(process.cwd(), 'coverage/coverage-summary.json');
|
||||||
|
if (existsSync(coveragePath)) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(readFileSync(coveragePath, 'utf-8'));
|
||||||
|
this.results.coverage = this.processCoverageResults(data);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading coverage results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processCoverageResults(data) {
|
||||||
|
const coverage = {
|
||||||
|
summary: {
|
||||||
|
lines: data.total.lines.pct,
|
||||||
|
statements: data.total.statements.pct,
|
||||||
|
functions: data.total.functions.pct,
|
||||||
|
branches: data.total.branches.pct,
|
||||||
|
average: 0
|
||||||
|
},
|
||||||
|
files: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Calculate average
|
||||||
|
coverage.summary.average = (
|
||||||
|
coverage.summary.lines +
|
||||||
|
coverage.summary.statements +
|
||||||
|
coverage.summary.functions +
|
||||||
|
coverage.summary.branches
|
||||||
|
) / 4;
|
||||||
|
|
||||||
|
// Process file coverage
|
||||||
|
for (const [filePath, fileData] of Object.entries(data)) {
|
||||||
|
if (filePath !== 'total') {
|
||||||
|
coverage.files.push({
|
||||||
|
path: filePath,
|
||||||
|
lines: fileData.lines.pct,
|
||||||
|
statements: fileData.statements.pct,
|
||||||
|
functions: fileData.functions.pct,
|
||||||
|
branches: fileData.branches.pct,
|
||||||
|
uncoveredLines: fileData.lines.total - fileData.lines.covered
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort files by coverage (lowest first)
|
||||||
|
coverage.files.sort((a, b) => a.lines - b.lines);
|
||||||
|
|
||||||
|
return coverage;
|
||||||
|
}
|
||||||
|
|
||||||
|
loadBenchmarkResults() {
|
||||||
|
const benchmarkPath = resolve(process.cwd(), 'benchmark-results.json');
|
||||||
|
if (existsSync(benchmarkPath)) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(readFileSync(benchmarkPath, 'utf-8'));
|
||||||
|
this.results.benchmarks = this.processBenchmarkResults(data);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading benchmark results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processBenchmarkResults(data) {
|
||||||
|
const benchmarks = {
|
||||||
|
timestamp: data.timestamp,
|
||||||
|
results: []
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const file of data.files || []) {
|
||||||
|
for (const group of file.groups || []) {
|
||||||
|
for (const benchmark of group.benchmarks || []) {
|
||||||
|
benchmarks.results.push({
|
||||||
|
file: file.filepath,
|
||||||
|
group: group.name,
|
||||||
|
name: benchmark.name,
|
||||||
|
ops: benchmark.result.hz,
|
||||||
|
mean: benchmark.result.mean,
|
||||||
|
min: benchmark.result.min,
|
||||||
|
max: benchmark.result.max,
|
||||||
|
p75: benchmark.result.p75,
|
||||||
|
p99: benchmark.result.p99,
|
||||||
|
samples: benchmark.result.samples
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by ops/sec (highest first)
|
||||||
|
benchmarks.results.sort((a, b) => b.ops - a.ops);
|
||||||
|
|
||||||
|
return benchmarks;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateMarkdownReport() {
|
||||||
|
let report = '# n8n-mcp Test Report\n\n';
|
||||||
|
report += `Generated: ${this.results.metadata.timestamp}\n\n`;
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
report += '## Build Information\n\n';
|
||||||
|
report += `- **Repository**: ${this.results.metadata.repository}\n`;
|
||||||
|
report += `- **Commit**: ${this.results.metadata.sha.substring(0, 7)}\n`;
|
||||||
|
report += `- **Branch**: ${this.results.metadata.branch}\n`;
|
||||||
|
report += `- **Run**: #${this.results.metadata.runNumber}\n\n`;
|
||||||
|
|
||||||
|
// Test Results
|
||||||
|
if (this.results.tests) {
|
||||||
|
const { summary, testSuites, failedTests } = this.results.tests;
|
||||||
|
const emoji = summary.success ? '✅' : '❌';
|
||||||
|
|
||||||
|
report += `## ${emoji} Test Results\n\n`;
|
||||||
|
report += `### Summary\n\n`;
|
||||||
|
report += `- **Total Tests**: ${summary.total}\n`;
|
||||||
|
report += `- **Passed**: ${summary.passed} (${((summary.passed / summary.total) * 100).toFixed(1)}%)\n`;
|
||||||
|
report += `- **Failed**: ${summary.failed}\n`;
|
||||||
|
report += `- **Skipped**: ${summary.skipped}\n`;
|
||||||
|
report += `- **Duration**: ${(summary.duration / 1000).toFixed(2)}s\n\n`;
|
||||||
|
|
||||||
|
// Test Suites
|
||||||
|
if (testSuites.length > 0) {
|
||||||
|
report += '### Test Suites\n\n';
|
||||||
|
report += '| Suite | Status | Tests | Duration |\n';
|
||||||
|
report += '|-------|--------|-------|----------|\n';
|
||||||
|
|
||||||
|
for (const suite of testSuites) {
|
||||||
|
const status = suite.status === 'passed' ? '✅' : '❌';
|
||||||
|
const tests = `${suite.tests.passed}/${suite.tests.total}`;
|
||||||
|
const duration = `${(suite.duration / 1000).toFixed(2)}s`;
|
||||||
|
report += `| ${suite.name} | ${status} | ${tests} | ${duration} |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failed Tests
|
||||||
|
if (failedTests.length > 0) {
|
||||||
|
report += '### Failed Tests\n\n';
|
||||||
|
for (const failed of failedTests) {
|
||||||
|
report += `#### ${failed.suite} > ${failed.test}\n\n`;
|
||||||
|
report += '```\n';
|
||||||
|
report += failed.error;
|
||||||
|
report += '\n```\n\n';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coverage Results
|
||||||
|
if (this.results.coverage) {
|
||||||
|
const { summary, files } = this.results.coverage;
|
||||||
|
const emoji = summary.average >= 80 ? '✅' : summary.average >= 60 ? '⚠️' : '❌';
|
||||||
|
|
||||||
|
report += `## ${emoji} Coverage Report\n\n`;
|
||||||
|
report += '### Summary\n\n';
|
||||||
|
report += `- **Lines**: ${summary.lines.toFixed(2)}%\n`;
|
||||||
|
report += `- **Statements**: ${summary.statements.toFixed(2)}%\n`;
|
||||||
|
report += `- **Functions**: ${summary.functions.toFixed(2)}%\n`;
|
||||||
|
report += `- **Branches**: ${summary.branches.toFixed(2)}%\n`;
|
||||||
|
report += `- **Average**: ${summary.average.toFixed(2)}%\n\n`;
|
||||||
|
|
||||||
|
// Files with low coverage
|
||||||
|
const lowCoverageFiles = files.filter(f => f.lines < 80).slice(0, 10);
|
||||||
|
if (lowCoverageFiles.length > 0) {
|
||||||
|
report += '### Files with Low Coverage\n\n';
|
||||||
|
report += '| File | Lines | Uncovered Lines |\n';
|
||||||
|
report += '|------|-------|----------------|\n';
|
||||||
|
|
||||||
|
for (const file of lowCoverageFiles) {
|
||||||
|
const fileName = file.path.split('/').pop();
|
||||||
|
report += `| ${fileName} | ${file.lines.toFixed(1)}% | ${file.uncoveredLines} |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Benchmark Results
|
||||||
|
if (this.results.benchmarks && this.results.benchmarks.results.length > 0) {
|
||||||
|
report += '## ⚡ Benchmark Results\n\n';
|
||||||
|
report += '### Top Performers\n\n';
|
||||||
|
report += '| Benchmark | Ops/sec | Mean (ms) | Samples |\n';
|
||||||
|
report += '|-----------|---------|-----------|----------|\n';
|
||||||
|
|
||||||
|
for (const bench of this.results.benchmarks.results.slice(0, 10)) {
|
||||||
|
const opsFormatted = bench.ops.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const meanFormatted = (bench.mean * 1000).toFixed(3);
|
||||||
|
report += `| ${bench.name} | ${opsFormatted} | ${meanFormatted} | ${bench.samples} |\n`;
|
||||||
|
}
|
||||||
|
report += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateJsonReport() {
|
||||||
|
return JSON.stringify(this.results, null, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
generateHtmlReport() {
|
||||||
|
const htmlTemplate = `<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>n8n-mcp Test Report</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
|
line-height: 1.6;
|
||||||
|
color: #333;
|
||||||
|
max-width: 1200px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 20px;
|
||||||
|
background-color: #f5f5f5;
|
||||||
|
}
|
||||||
|
.header {
|
||||||
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||||
|
color: white;
|
||||||
|
padding: 30px;
|
||||||
|
border-radius: 10px;
|
||||||
|
margin-bottom: 30px;
|
||||||
|
}
|
||||||
|
.header h1 {
|
||||||
|
margin: 0 0 10px 0;
|
||||||
|
font-size: 2.5em;
|
||||||
|
}
|
||||||
|
.metadata {
|
||||||
|
opacity: 0.9;
|
||||||
|
font-size: 0.9em;
|
||||||
|
}
|
||||||
|
.section {
|
||||||
|
background: white;
|
||||||
|
padding: 25px;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
border-radius: 10px;
|
||||||
|
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
||||||
|
}
|
||||||
|
.section h2 {
|
||||||
|
margin-top: 0;
|
||||||
|
color: #333;
|
||||||
|
border-bottom: 2px solid #eee;
|
||||||
|
padding-bottom: 10px;
|
||||||
|
}
|
||||||
|
.stats {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||||
|
gap: 20px;
|
||||||
|
margin: 20px 0;
|
||||||
|
}
|
||||||
|
.stat-card {
|
||||||
|
background: #f8f9fa;
|
||||||
|
padding: 20px;
|
||||||
|
border-radius: 8px;
|
||||||
|
text-align: center;
|
||||||
|
border: 1px solid #e9ecef;
|
||||||
|
}
|
||||||
|
.stat-card .value {
|
||||||
|
font-size: 2em;
|
||||||
|
font-weight: bold;
|
||||||
|
color: #667eea;
|
||||||
|
}
|
||||||
|
.stat-card .label {
|
||||||
|
color: #666;
|
||||||
|
font-size: 0.9em;
|
||||||
|
margin-top: 5px;
|
||||||
|
}
|
||||||
|
table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
margin: 20px 0;
|
||||||
|
}
|
||||||
|
th, td {
|
||||||
|
padding: 12px;
|
||||||
|
text-align: left;
|
||||||
|
border-bottom: 1px solid #ddd;
|
||||||
|
}
|
||||||
|
th {
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
font-weight: 600;
|
||||||
|
color: #495057;
|
||||||
|
}
|
||||||
|
tr:hover {
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
}
|
||||||
|
.success { color: #28a745; }
|
||||||
|
.warning { color: #ffc107; }
|
||||||
|
.danger { color: #dc3545; }
|
||||||
|
.failed-test {
|
||||||
|
background-color: #fff5f5;
|
||||||
|
border: 1px solid #feb2b2;
|
||||||
|
border-radius: 5px;
|
||||||
|
padding: 15px;
|
||||||
|
margin: 10px 0;
|
||||||
|
}
|
||||||
|
.failed-test h4 {
|
||||||
|
margin: 0 0 10px 0;
|
||||||
|
color: #c53030;
|
||||||
|
}
|
||||||
|
.error-message {
|
||||||
|
background-color: #1a202c;
|
||||||
|
color: #e2e8f0;
|
||||||
|
padding: 15px;
|
||||||
|
border-radius: 5px;
|
||||||
|
font-family: 'Courier New', monospace;
|
||||||
|
font-size: 0.9em;
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
.progress-bar {
|
||||||
|
width: 100%;
|
||||||
|
height: 20px;
|
||||||
|
background-color: #e9ecef;
|
||||||
|
border-radius: 10px;
|
||||||
|
overflow: hidden;
|
||||||
|
margin: 10px 0;
|
||||||
|
}
|
||||||
|
.progress-fill {
|
||||||
|
height: 100%;
|
||||||
|
background: linear-gradient(90deg, #28a745 0%, #20c997 100%);
|
||||||
|
transition: width 0.3s ease;
|
||||||
|
}
|
||||||
|
.coverage-low { background: linear-gradient(90deg, #dc3545 0%, #f86734 100%); }
|
||||||
|
.coverage-medium { background: linear-gradient(90deg, #ffc107 0%, #ffb347 100%); }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="header">
|
||||||
|
<h1>n8n-mcp Test Report</h1>
|
||||||
|
<div class="metadata">
|
||||||
|
<div>Repository: ${this.results.metadata.repository}</div>
|
||||||
|
<div>Commit: ${this.results.metadata.sha.substring(0, 7)}</div>
|
||||||
|
<div>Run: #${this.results.metadata.runNumber}</div>
|
||||||
|
<div>Generated: ${new Date(this.results.metadata.timestamp).toLocaleString()}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
${this.generateTestResultsHtml()}
|
||||||
|
${this.generateCoverageHtml()}
|
||||||
|
${this.generateBenchmarkHtml()}
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
return htmlTemplate;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateTestResultsHtml() {
|
||||||
|
if (!this.results.tests) return '';
|
||||||
|
|
||||||
|
const { summary, testSuites, failedTests } = this.results.tests;
|
||||||
|
const successRate = ((summary.passed / summary.total) * 100).toFixed(1);
|
||||||
|
const statusClass = summary.success ? 'success' : 'danger';
|
||||||
|
const statusIcon = summary.success ? '✅' : '❌';
|
||||||
|
|
||||||
|
let html = `
|
||||||
|
<div class="section">
|
||||||
|
<h2>${statusIcon} Test Results</h2>
|
||||||
|
<div class="stats">
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${summary.total}</div>
|
||||||
|
<div class="label">Total Tests</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value ${statusClass}">${summary.passed}</div>
|
||||||
|
<div class="label">Passed</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value ${summary.failed > 0 ? 'danger' : ''}">${summary.failed}</div>
|
||||||
|
<div class="label">Failed</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${successRate}%</div>
|
||||||
|
<div class="label">Success Rate</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${(summary.duration / 1000).toFixed(1)}s</div>
|
||||||
|
<div class="label">Duration</div>
|
||||||
|
</div>
|
||||||
|
</div>`;
|
||||||
|
|
||||||
|
if (testSuites.length > 0) {
|
||||||
|
html += `
|
||||||
|
<h3>Test Suites</h3>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Suite</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Tests</th>
|
||||||
|
<th>Duration</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>`;
|
||||||
|
|
||||||
|
for (const suite of testSuites) {
|
||||||
|
const status = suite.status === 'passed' ? '✅' : '❌';
|
||||||
|
const statusClass = suite.status === 'passed' ? 'success' : 'danger';
|
||||||
|
html += `
|
||||||
|
<tr>
|
||||||
|
<td>${suite.name}</td>
|
||||||
|
<td class="${statusClass}">${status}</td>
|
||||||
|
<td>${suite.tests.passed}/${suite.tests.total}</td>
|
||||||
|
<td>${(suite.duration / 1000).toFixed(2)}s</td>
|
||||||
|
</tr>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `
|
||||||
|
</tbody>
|
||||||
|
</table>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failedTests.length > 0) {
|
||||||
|
html += `
|
||||||
|
<h3>Failed Tests</h3>`;
|
||||||
|
|
||||||
|
for (const failed of failedTests) {
|
||||||
|
html += `
|
||||||
|
<div class="failed-test">
|
||||||
|
<h4>${failed.suite} > ${failed.test}</h4>
|
||||||
|
<div class="error-message">${this.escapeHtml(failed.error)}</div>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `</div>`;
|
||||||
|
return html;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateCoverageHtml() {
|
||||||
|
if (!this.results.coverage) return '';
|
||||||
|
|
||||||
|
const { summary, files } = this.results.coverage;
|
||||||
|
const coverageClass = summary.average >= 80 ? 'success' : summary.average >= 60 ? 'warning' : 'danger';
|
||||||
|
const progressClass = summary.average >= 80 ? '' : summary.average >= 60 ? 'coverage-medium' : 'coverage-low';
|
||||||
|
|
||||||
|
let html = `
|
||||||
|
<div class="section">
|
||||||
|
<h2>📊 Coverage Report</h2>
|
||||||
|
<div class="stats">
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value ${coverageClass}">${summary.average.toFixed(1)}%</div>
|
||||||
|
<div class="label">Average Coverage</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${summary.lines.toFixed(1)}%</div>
|
||||||
|
<div class="label">Lines</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${summary.statements.toFixed(1)}%</div>
|
||||||
|
<div class="label">Statements</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${summary.functions.toFixed(1)}%</div>
|
||||||
|
<div class="label">Functions</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="value">${summary.branches.toFixed(1)}%</div>
|
||||||
|
<div class="label">Branches</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="progress-bar">
|
||||||
|
<div class="progress-fill ${progressClass}" style="width: ${summary.average}%"></div>
|
||||||
|
</div>`;
|
||||||
|
|
||||||
|
const lowCoverageFiles = files.filter(f => f.lines < 80).slice(0, 10);
|
||||||
|
if (lowCoverageFiles.length > 0) {
|
||||||
|
html += `
|
||||||
|
<h3>Files with Low Coverage</h3>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>File</th>
|
||||||
|
<th>Lines</th>
|
||||||
|
<th>Statements</th>
|
||||||
|
<th>Functions</th>
|
||||||
|
<th>Branches</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>`;
|
||||||
|
|
||||||
|
for (const file of lowCoverageFiles) {
|
||||||
|
const fileName = file.path.split('/').pop();
|
||||||
|
html += `
|
||||||
|
<tr>
|
||||||
|
<td>${fileName}</td>
|
||||||
|
<td class="${file.lines < 50 ? 'danger' : file.lines < 80 ? 'warning' : ''}">${file.lines.toFixed(1)}%</td>
|
||||||
|
<td>${file.statements.toFixed(1)}%</td>
|
||||||
|
<td>${file.functions.toFixed(1)}%</td>
|
||||||
|
<td>${file.branches.toFixed(1)}%</td>
|
||||||
|
</tr>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `
|
||||||
|
</tbody>
|
||||||
|
</table>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `</div>`;
|
||||||
|
return html;
|
||||||
|
}
|
||||||
|
|
||||||
|
generateBenchmarkHtml() {
|
||||||
|
if (!this.results.benchmarks || this.results.benchmarks.results.length === 0) return '';
|
||||||
|
|
||||||
|
let html = `
|
||||||
|
<div class="section">
|
||||||
|
<h2>⚡ Benchmark Results</h2>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Benchmark</th>
|
||||||
|
<th>Operations/sec</th>
|
||||||
|
<th>Mean Time (ms)</th>
|
||||||
|
<th>Min (ms)</th>
|
||||||
|
<th>Max (ms)</th>
|
||||||
|
<th>Samples</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>`;
|
||||||
|
|
||||||
|
for (const bench of this.results.benchmarks.results.slice(0, 20)) {
|
||||||
|
const opsFormatted = bench.ops.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const meanFormatted = (bench.mean * 1000).toFixed(3);
|
||||||
|
const minFormatted = (bench.min * 1000).toFixed(3);
|
||||||
|
const maxFormatted = (bench.max * 1000).toFixed(3);
|
||||||
|
|
||||||
|
html += `
|
||||||
|
<tr>
|
||||||
|
<td>${bench.name}</td>
|
||||||
|
<td><strong>${opsFormatted}</strong></td>
|
||||||
|
<td>${meanFormatted}</td>
|
||||||
|
<td>${minFormatted}</td>
|
||||||
|
<td>${maxFormatted}</td>
|
||||||
|
<td>${bench.samples}</td>
|
||||||
|
</tr>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `
|
||||||
|
</tbody>
|
||||||
|
</table>`;
|
||||||
|
|
||||||
|
if (this.results.benchmarks.results.length > 20) {
|
||||||
|
html += `<p><em>Showing top 20 of ${this.results.benchmarks.results.length} benchmarks</em></p>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `</div>`;
|
||||||
|
return html;
|
||||||
|
}
|
||||||
|
|
||||||
|
escapeHtml(text) {
|
||||||
|
const map = {
|
||||||
|
'&': '&',
|
||||||
|
'<': '<',
|
||||||
|
'>': '>',
|
||||||
|
'"': '"',
|
||||||
|
"'": '''
|
||||||
|
};
|
||||||
|
return text.replace(/[&<>"']/g, m => map[m]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async generate() {
|
||||||
|
// Load all results
|
||||||
|
this.loadTestResults();
|
||||||
|
this.loadCoverageResults();
|
||||||
|
this.loadBenchmarkResults();
|
||||||
|
|
||||||
|
// Ensure output directory exists
|
||||||
|
const outputDir = resolve(process.cwd(), 'test-reports');
|
||||||
|
if (!existsSync(outputDir)) {
|
||||||
|
mkdirSync(outputDir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate reports in different formats
|
||||||
|
const markdownReport = this.generateMarkdownReport();
|
||||||
|
const jsonReport = this.generateJsonReport();
|
||||||
|
const htmlReport = this.generateHtmlReport();
|
||||||
|
|
||||||
|
// Write reports
|
||||||
|
writeFileSync(resolve(outputDir, 'report.md'), markdownReport);
|
||||||
|
writeFileSync(resolve(outputDir, 'report.json'), jsonReport);
|
||||||
|
writeFileSync(resolve(outputDir, 'report.html'), htmlReport);
|
||||||
|
|
||||||
|
console.log('Test reports generated successfully:');
|
||||||
|
console.log('- test-reports/report.md');
|
||||||
|
console.log('- test-reports/report.json');
|
||||||
|
console.log('- test-reports/report.html');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the generator
|
||||||
|
const generator = new TestReportGenerator();
|
||||||
|
generator.generate().catch(console.error);
|
||||||
167
scripts/generate-test-summary.js
Normal file
167
scripts/generate-test-summary.js
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { readFileSync, existsSync } from 'fs';
|
||||||
|
import { resolve } from 'path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a markdown summary of test results for PR comments
|
||||||
|
*/
|
||||||
|
function generateTestSummary() {
|
||||||
|
const results = {
|
||||||
|
tests: null,
|
||||||
|
coverage: null,
|
||||||
|
benchmarks: null,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Read test results
|
||||||
|
const testResultPath = resolve(process.cwd(), 'test-results/results.json');
|
||||||
|
if (existsSync(testResultPath)) {
|
||||||
|
try {
|
||||||
|
const testData = JSON.parse(readFileSync(testResultPath, 'utf-8'));
|
||||||
|
const totalTests = testData.numTotalTests || 0;
|
||||||
|
const passedTests = testData.numPassedTests || 0;
|
||||||
|
const failedTests = testData.numFailedTests || 0;
|
||||||
|
const skippedTests = testData.numSkippedTests || 0;
|
||||||
|
const duration = testData.duration || 0;
|
||||||
|
|
||||||
|
results.tests = {
|
||||||
|
total: totalTests,
|
||||||
|
passed: passedTests,
|
||||||
|
failed: failedTests,
|
||||||
|
skipped: skippedTests,
|
||||||
|
duration: duration,
|
||||||
|
success: failedTests === 0
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error reading test results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read coverage results
|
||||||
|
const coveragePath = resolve(process.cwd(), 'coverage/coverage-summary.json');
|
||||||
|
if (existsSync(coveragePath)) {
|
||||||
|
try {
|
||||||
|
const coverageData = JSON.parse(readFileSync(coveragePath, 'utf-8'));
|
||||||
|
const total = coverageData.total;
|
||||||
|
|
||||||
|
results.coverage = {
|
||||||
|
lines: total.lines.pct,
|
||||||
|
statements: total.statements.pct,
|
||||||
|
functions: total.functions.pct,
|
||||||
|
branches: total.branches.pct
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error reading coverage results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read benchmark results
|
||||||
|
const benchmarkPath = resolve(process.cwd(), 'benchmark-results.json');
|
||||||
|
if (existsSync(benchmarkPath)) {
|
||||||
|
try {
|
||||||
|
const benchmarkData = JSON.parse(readFileSync(benchmarkPath, 'utf-8'));
|
||||||
|
const benchmarks = [];
|
||||||
|
|
||||||
|
for (const file of benchmarkData.files || []) {
|
||||||
|
for (const group of file.groups || []) {
|
||||||
|
for (const benchmark of group.benchmarks || []) {
|
||||||
|
benchmarks.push({
|
||||||
|
name: `${group.name} - ${benchmark.name}`,
|
||||||
|
mean: benchmark.result.mean,
|
||||||
|
ops: benchmark.result.hz
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
results.benchmarks = benchmarks;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error reading benchmark results:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate markdown summary
|
||||||
|
let summary = '## Test Results Summary\n\n';
|
||||||
|
|
||||||
|
// Test results
|
||||||
|
if (results.tests) {
|
||||||
|
const { total, passed, failed, skipped, duration, success } = results.tests;
|
||||||
|
const emoji = success ? '✅' : '❌';
|
||||||
|
const status = success ? 'PASSED' : 'FAILED';
|
||||||
|
|
||||||
|
summary += `### ${emoji} Tests ${status}\n\n`;
|
||||||
|
summary += `| Metric | Value |\n`;
|
||||||
|
summary += `|--------|-------|\n`;
|
||||||
|
summary += `| Total Tests | ${total} |\n`;
|
||||||
|
summary += `| Passed | ${passed} |\n`;
|
||||||
|
summary += `| Failed | ${failed} |\n`;
|
||||||
|
summary += `| Skipped | ${skipped} |\n`;
|
||||||
|
summary += `| Duration | ${(duration / 1000).toFixed(2)}s |\n\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coverage results
|
||||||
|
if (results.coverage) {
|
||||||
|
const { lines, statements, functions, branches } = results.coverage;
|
||||||
|
const avgCoverage = (lines + statements + functions + branches) / 4;
|
||||||
|
const emoji = avgCoverage >= 80 ? '✅' : avgCoverage >= 60 ? '⚠️' : '❌';
|
||||||
|
|
||||||
|
summary += `### ${emoji} Coverage Report\n\n`;
|
||||||
|
summary += `| Type | Coverage |\n`;
|
||||||
|
summary += `|------|----------|\n`;
|
||||||
|
summary += `| Lines | ${lines.toFixed(2)}% |\n`;
|
||||||
|
summary += `| Statements | ${statements.toFixed(2)}% |\n`;
|
||||||
|
summary += `| Functions | ${functions.toFixed(2)}% |\n`;
|
||||||
|
summary += `| Branches | ${branches.toFixed(2)}% |\n`;
|
||||||
|
summary += `| **Average** | **${avgCoverage.toFixed(2)}%** |\n\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Benchmark results
|
||||||
|
if (results.benchmarks && results.benchmarks.length > 0) {
|
||||||
|
summary += `### ⚡ Benchmark Results\n\n`;
|
||||||
|
summary += `| Benchmark | Ops/sec | Mean (ms) |\n`;
|
||||||
|
summary += `|-----------|---------|------------|\n`;
|
||||||
|
|
||||||
|
for (const bench of results.benchmarks.slice(0, 10)) { // Show top 10
|
||||||
|
const opsFormatted = bench.ops.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||||
|
const meanFormatted = (bench.mean * 1000).toFixed(3);
|
||||||
|
summary += `| ${bench.name} | ${opsFormatted} | ${meanFormatted} |\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (results.benchmarks.length > 10) {
|
||||||
|
summary += `\n*...and ${results.benchmarks.length - 10} more benchmarks*\n`;
|
||||||
|
}
|
||||||
|
summary += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Links to artifacts
|
||||||
|
const runId = process.env.GITHUB_RUN_ID;
|
||||||
|
const runNumber = process.env.GITHUB_RUN_NUMBER;
|
||||||
|
const sha = process.env.GITHUB_SHA;
|
||||||
|
|
||||||
|
if (runId) {
|
||||||
|
summary += `### 📊 Artifacts\n\n`;
|
||||||
|
summary += `- 📄 [Test Results](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId})\n`;
|
||||||
|
summary += `- 📊 [Coverage Report](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId})\n`;
|
||||||
|
summary += `- ⚡ [Benchmark Results](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId})\n\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
summary += `---\n`;
|
||||||
|
summary += `*Generated at ${new Date().toUTCString()}*\n`;
|
||||||
|
if (sha) {
|
||||||
|
summary += `*Commit: ${sha.substring(0, 7)}*\n`;
|
||||||
|
}
|
||||||
|
if (runNumber) {
|
||||||
|
summary += `*Run: #${runNumber}*\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate and output summary
|
||||||
|
const summary = generateTestSummary();
|
||||||
|
console.log(summary);
|
||||||
|
|
||||||
|
// Also write to file for artifact
|
||||||
|
import { writeFileSync } from 'fs';
|
||||||
|
writeFileSync('test-summary.md', summary);
|
||||||
@@ -10,8 +10,9 @@ export default defineConfig({
|
|||||||
// Benchmark specific options
|
// Benchmark specific options
|
||||||
include: ['tests/benchmarks/**/*.bench.ts'],
|
include: ['tests/benchmarks/**/*.bench.ts'],
|
||||||
reporters: process.env.CI
|
reporters: process.env.CI
|
||||||
? [['./scripts/vitest-benchmark-json-reporter.js', {}]]
|
? ['default', ['./scripts/vitest-benchmark-json-reporter.js', {}]]
|
||||||
: ['default'],
|
: ['default'],
|
||||||
|
outputFile: './benchmark-results.json',
|
||||||
},
|
},
|
||||||
setupFiles: [],
|
setupFiles: [],
|
||||||
pool: 'forks',
|
pool: 'forks',
|
||||||
|
|||||||
@@ -22,10 +22,11 @@ export default defineConfig({
|
|||||||
// Retry configuration
|
// Retry configuration
|
||||||
retry: parseInt(process.env.TEST_RETRY_ATTEMPTS || '2', 10),
|
retry: parseInt(process.env.TEST_RETRY_ATTEMPTS || '2', 10),
|
||||||
// Test reporter
|
// Test reporter
|
||||||
reporters: process.env.CI ? ['default', 'json', 'junit'] : ['default'],
|
reporters: process.env.CI ? ['default', 'json', 'junit', 'html'] : ['default'],
|
||||||
outputFile: {
|
outputFile: {
|
||||||
json: './test-results/results.json',
|
json: './test-results/results.json',
|
||||||
junit: './test-results/junit.xml'
|
junit: './test-results/junit.xml',
|
||||||
|
html: './test-results/html/index.html'
|
||||||
},
|
},
|
||||||
coverage: {
|
coverage: {
|
||||||
provider: 'v8',
|
provider: 'v8',
|
||||||
|
|||||||
Reference in New Issue
Block a user