diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index f347c33..e8ab8ad 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -93,71 +93,84 @@ jobs: - name: Post benchmark comparison to PR if: always() uses: actions/github-script@v7 + continue-on-error: true with: script: | - const fs = require('fs'); - let comment = '## āš” Benchmark Comparison\n\n'; - try { - if (fs.existsSync('benchmark-comparison.md')) { - const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8'); - comment += comparison; - } else { - comment += 'Benchmark comparison could not be generated.'; + const fs = require('fs'); + let comment = '## āš” Benchmark Comparison\n\n'; + + try { + if (fs.existsSync('benchmark-comparison.md')) { + const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8'); + comment += comparison; + } else { + comment += 'Benchmark comparison could not be generated.'; + } + } catch (error) { + comment += `Error reading benchmark comparison: ${error.message}`; } - } catch (error) { - comment += `Error reading benchmark comparison: ${error.message}`; - } - - comment += '\n\n---\n'; - comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`; - - // Find existing comment - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const botComment = comments.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('## āš” Benchmark Comparison') - ); - - if (botComment) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: comment - }); - } else { - await github.rest.issues.createComment({ + + comment += '\n\n---\n'; + comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: comment }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('## āš” Benchmark Comparison') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: comment + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: comment + }); + } + } catch (error) { + console.error('Failed to create/update PR comment:', error.message); + console.log('This is likely due to insufficient permissions for external PRs.'); + console.log('Benchmark comparison has been saved to artifacts instead.'); } # Add status check - name: Set benchmark status if: always() uses: actions/github-script@v7 + continue-on-error: true with: script: | - const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true'; - const state = hasRegression ? 'failure' : 'success'; - const description = hasRegression - ? 'Performance regressions detected' - : 'No performance regressions'; - - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: state, - target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`, - description: description, - context: 'benchmarks/regression-check' - }); \ No newline at end of file + try { + const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true'; + const state = hasRegression ? 'failure' : 'success'; + const description = hasRegression + ? 'Performance regressions detected' + : 'No performance regressions'; + + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: state, + target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`, + description: description, + context: 'benchmarks/regression-check' + }); + } catch (error) { + console.error('Failed to create commit status:', error.message); + console.log('This is likely due to insufficient permissions for external PRs.'); + } \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7a356cd..628a1f4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -103,12 +103,14 @@ jobs: # Store benchmark results and compare - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 + continue-on-error: true + id: benchmark with: name: n8n-mcp Benchmarks tool: 'customSmallerIsBetter' output-file-path: benchmark-results-formatted.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Where to store benchmark data benchmark-data-dir-path: 'benchmarks' # Alert when performance regresses by 10% @@ -120,52 +122,60 @@ jobs: summary-always: true # Max number of data points to retain max-items-in-chart: 50 + fail-on-alert: false # Comment on PR with benchmark results - name: Comment PR with results uses: actions/github-script@v7 if: github.event_name == 'pull_request' + continue-on-error: true with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const fs = require('fs'); - const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8')); - - // Format results for PR comment - let comment = '## šŸ“Š Performance Benchmark Results\n\n'; - comment += `šŸ• Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`; - comment += '| Benchmark | Time | Ops/sec | Range |\n'; - comment += '|-----------|------|---------|-------|\n'; - - // Group benchmarks by category - const categories = {}; - for (const benchmark of summary.benchmarks) { - const [category, ...nameParts] = benchmark.name.split(' - '); - if (!categories[category]) categories[category] = []; - categories[category].push({ - ...benchmark, - shortName: nameParts.join(' - ') - }); - } - - // Display by category - for (const [category, benchmarks] of Object.entries(categories)) { - comment += `\n### ${category}\n`; - for (const benchmark of benchmarks) { - comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`; + try { + const fs = require('fs'); + const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8')); + + // Format results for PR comment + let comment = '## šŸ“Š Performance Benchmark Results\n\n'; + comment += `šŸ• Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`; + comment += '| Benchmark | Time | Ops/sec | Range |\n'; + comment += '|-----------|------|---------|-------|\n'; + + // Group benchmarks by category + const categories = {}; + for (const benchmark of summary.benchmarks) { + const [category, ...nameParts] = benchmark.name.split(' - '); + if (!categories[category]) categories[category] = []; + categories[category].push({ + ...benchmark, + shortName: nameParts.join(' - ') + }); } + + // Display by category + for (const [category, benchmarks] of Object.entries(categories)) { + comment += `\n### ${category}\n`; + for (const benchmark of benchmarks) { + comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`; + } + } + + // Add comparison link + comment += '\n\nšŸ“ˆ [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n'; + comment += '\nāš” Performance regressions >10% will be flagged automatically.\n'; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } catch (error) { + console.error('Failed to create PR comment:', error.message); + console.log('This is likely due to insufficient permissions for external PRs.'); + console.log('Benchmark results have been saved to artifacts instead.'); } - - // Add comparison link - comment += '\n\nšŸ“ˆ [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n'; - comment += '\nāš” Performance regressions >10% will be flagged automatically.\n'; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); # Deploy benchmark results to GitHub Pages deploy: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f93bdf5..0200ec1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,7 @@ jobs: - name: Create test report comment if: github.event_name == 'pull_request' && always() uses: actions/github-script@v7 + continue-on-error: true with: script: | const fs = require('fs'); @@ -161,34 +162,40 @@ jobs: console.error('Error reading test summary:', error); } - // Find existing comment - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const botComment = comments.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('## Test Results') - ); - - if (botComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: summary - }); - } else { - // Create new comment - await github.rest.issues.createComment({ + try { + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: summary }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('## Test Results') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: summary + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: summary + }); + } + } catch (error) { + console.error('Failed to create/update PR comment:', error.message); + console.log('This is likely due to insufficient permissions for external PRs.'); + console.log('Test results have been saved to the job summary instead.'); } # Generate job summary @@ -260,11 +267,13 @@ jobs: - name: Publish test results uses: dorny/test-reporter@v1 if: always() + continue-on-error: true with: name: Test Results path: 'artifacts/test-results-*/test-results/junit.xml' reporter: java-junit fail-on-error: false + fail-on-empty: false # Create a combined artifact with all results - name: Create combined results artifact