feat: add comprehensive performance benchmark tracking system
- Create benchmark test suites for critical operations: - Node loading performance - Database query performance - Search operations performance - Validation performance - MCP tool execution performance - Add GitHub Actions workflow for benchmark tracking: - Runs on push to main and PRs - Uses github-action-benchmark for historical tracking - Comments on PRs with performance results - Alerts on >10% performance regressions - Stores results in GitHub Pages - Create benchmark infrastructure: - Custom Vitest benchmark configuration - JSON reporter for CI results - Result formatter for github-action-benchmark - Performance threshold documentation - Add supporting utilities: - SQLiteStorageService for benchmark database setup - MCPEngine wrapper for testing MCP tools - Test factories for generating benchmark data - Enhanced NodeRepository with benchmark methods - Document benchmark system: - Comprehensive benchmark guide in docs/BENCHMARKS.md - Performance thresholds in .github/BENCHMARK_THRESHOLDS.md - README for benchmarks directory - Integration with existing test suite The benchmark system will help monitor performance over time and catch regressions before they reach production. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
146
.github/workflows/benchmark.yml
vendored
Normal file
146
.github/workflows/benchmark.yml
vendored
Normal file
@@ -0,0 +1,146 @@
|
||||
name: Performance Benchmarks
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, feat/comprehensive-testing-suite]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
# For PR comments
|
||||
pull-requests: write
|
||||
# For pushing to gh-pages branch
|
||||
contents: write
|
||||
# For deployment to GitHub Pages
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
benchmark:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Fetch all history for proper benchmark comparison
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build project
|
||||
run: npm run build
|
||||
|
||||
- name: Run benchmarks
|
||||
run: npm run benchmark:ci
|
||||
|
||||
- name: Format benchmark results
|
||||
run: node scripts/format-benchmark-results.js
|
||||
|
||||
- name: Upload benchmark artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: benchmark-results
|
||||
path: |
|
||||
benchmark-results.json
|
||||
benchmark-results-formatted.json
|
||||
benchmark-summary.json
|
||||
|
||||
# Store benchmark results and compare
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: n8n-mcp Benchmarks
|
||||
tool: 'customSmallerIsBetter'
|
||||
output-file-path: benchmark-results-formatted.json
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Where to store benchmark data
|
||||
benchmark-data-dir-path: 'benchmarks'
|
||||
# Alert when performance regresses by 10%
|
||||
alert-threshold: '110%'
|
||||
# Comment on PR when regression is detected
|
||||
comment-on-alert: true
|
||||
alert-comment-cc-users: '@czlonkowski'
|
||||
# Summary always
|
||||
summary-always: true
|
||||
# Max number of data points to retain
|
||||
max-items-in-chart: 50
|
||||
|
||||
# Comment on PR with benchmark results
|
||||
- name: Comment PR with results
|
||||
uses: actions/github-script@v7
|
||||
if: github.event_name == 'pull_request'
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
|
||||
|
||||
// Format results for PR comment
|
||||
let comment = '## 📊 Performance Benchmark Results\n\n';
|
||||
comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
|
||||
comment += '| Benchmark | Time | Ops/sec | Range |\n';
|
||||
comment += '|-----------|------|---------|-------|\n';
|
||||
|
||||
// Group benchmarks by category
|
||||
const categories = {};
|
||||
for (const benchmark of summary.benchmarks) {
|
||||
const [category, ...nameParts] = benchmark.name.split(' - ');
|
||||
if (!categories[category]) categories[category] = [];
|
||||
categories[category].push({
|
||||
...benchmark,
|
||||
shortName: nameParts.join(' - ')
|
||||
});
|
||||
}
|
||||
|
||||
// Display by category
|
||||
for (const [category, benchmarks] of Object.entries(categories)) {
|
||||
comment += `\n### ${category}\n`;
|
||||
for (const benchmark of benchmarks) {
|
||||
comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// Add comparison link
|
||||
comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
|
||||
comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
|
||||
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
|
||||
# Deploy benchmark results to GitHub Pages
|
||||
deploy:
|
||||
needs: benchmark
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: gh-pages
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
|
||||
- name: Upload Pages artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: '.'
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
Reference in New Issue
Block a user