test: verify CI permission fixes handle external PR permissions gracefully

.github/workflows/benchmark-pr.yml

@@ -93,71 +93,84 @@ jobs:
       - name: Post benchmark comparison to PR
         if: always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
-            const fs = require('fs');
-            let comment = '## ⚡ Benchmark Comparison\n\n';
-            
             try {
-              if (fs.existsSync('benchmark-comparison.md')) {
-                const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
-                comment += comparison;
-              } else {
-                comment += 'Benchmark comparison could not be generated.';
+              const fs = require('fs');
+              let comment = '## ⚡ Benchmark Comparison\n\n';
+              
+              try {
+                if (fs.existsSync('benchmark-comparison.md')) {
+                  const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
+                  comment += comparison;
+                } else {
+                  comment += 'Benchmark comparison could not be generated.';
+                }
+              } catch (error) {
+                comment += `Error reading benchmark comparison: ${error.message}`;
               }
-            } catch (error) {
-              comment += `Error reading benchmark comparison: ${error.message}`;
-            }
-            
-            comment += '\n\n---\n';
-            comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
-            
-            // Find existing comment
-            const { data: comments } = await github.rest.issues.listComments({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-            });
-            
-            const botComment = comments.find(comment => 
-              comment.user.type === 'Bot' && 
-              comment.body.includes('## ⚡ Benchmark Comparison')
-            );
-            
-            if (botComment) {
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: botComment.id,
-                body: comment
-              });
-            } else {
-              await github.rest.issues.createComment({
+              
+              comment += '\n\n---\n';
+              comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
+              
+              // Find existing comment
+              const { data: comments } = await github.rest.issues.listComments({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 issue_number: context.issue.number,
-                body: comment
               });
+              
+              const botComment = comments.find(comment => 
+                comment.user.type === 'Bot' && 
+                comment.body.includes('## ⚡ Benchmark Comparison')
+              );
+              
+              if (botComment) {
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: botComment.id,
+                  body: comment
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: comment
+                });
+              }
+            } catch (error) {
+              console.error('Failed to create/update PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Benchmark comparison has been saved to artifacts instead.');
             }
       
       # Add status check
       - name: Set benchmark status
         if: always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
-            const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
-            const state = hasRegression ? 'failure' : 'success';
-            const description = hasRegression 
-              ? 'Performance regressions detected' 
-              : 'No performance regressions';
-            
-            await github.rest.repos.createCommitStatus({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              sha: context.sha,
-              state: state,
-              target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
-              description: description,
-              context: 'benchmarks/regression-check'
-            });
+            try {
+              const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
+              const state = hasRegression ? 'failure' : 'success';
+              const description = hasRegression 
+                ? 'Performance regressions detected' 
+                : 'No performance regressions';
+              
+              await github.rest.repos.createCommitStatus({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                sha: context.sha,
+                state: state,
+                target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                description: description,
+                context: 'benchmarks/regression-check'
+              });
+            } catch (error) {
+              console.error('Failed to create commit status:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+            }

.github/workflows/benchmark.yml

@@ -103,12 +103,14 @@ jobs:
       # Store benchmark results and compare
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
+        continue-on-error: true
+        id: benchmark
         with:
           name: n8n-mcp Benchmarks
           tool: 'customSmallerIsBetter'
           output-file-path: benchmark-results-formatted.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          auto-push: true
+          auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
           # Where to store benchmark data
           benchmark-data-dir-path: 'benchmarks'
           # Alert when performance regresses by 10%
@@ -120,52 +122,60 @@ jobs:
           summary-always: true
           # Max number of data points to retain
           max-items-in-chart: 50
+          fail-on-alert: false
 
       # Comment on PR with benchmark results
       - name: Comment PR with results
         uses: actions/github-script@v7
         if: github.event_name == 'pull_request'
+        continue-on-error: true
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const fs = require('fs');
-            const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
-            
-            // Format results for PR comment
-            let comment = '## 📊 Performance Benchmark Results\n\n';
-            comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
-            comment += '| Benchmark | Time | Ops/sec | Range |\n';
-            comment += '|-----------|------|---------|-------|\n';
-            
-            // Group benchmarks by category
-            const categories = {};
-            for (const benchmark of summary.benchmarks) {
-              const [category, ...nameParts] = benchmark.name.split(' - ');
-              if (!categories[category]) categories[category] = [];
-              categories[category].push({
-                ...benchmark,
-                shortName: nameParts.join(' - ')
-              });
-            }
-            
-            // Display by category
-            for (const [category, benchmarks] of Object.entries(categories)) {
-              comment += `\n### ${category}\n`;
-              for (const benchmark of benchmarks) {
-                comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
+            try {
+              const fs = require('fs');
+              const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
+              
+              // Format results for PR comment
+              let comment = '## 📊 Performance Benchmark Results\n\n';
+              comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
+              comment += '| Benchmark | Time | Ops/sec | Range |\n';
+              comment += '|-----------|------|---------|-------|\n';
+              
+              // Group benchmarks by category
+              const categories = {};
+              for (const benchmark of summary.benchmarks) {
+                const [category, ...nameParts] = benchmark.name.split(' - ');
+                if (!categories[category]) categories[category] = [];
+                categories[category].push({
+                  ...benchmark,
+                  shortName: nameParts.join(' - ')
+                });
               }
+              
+              // Display by category
+              for (const [category, benchmarks] of Object.entries(categories)) {
+                comment += `\n### ${category}\n`;
+                for (const benchmark of benchmarks) {
+                  comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
+                }
+              }
+              
+              // Add comparison link
+              comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
+              comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
+              
+              await github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: comment
+              });
+            } catch (error) {
+              console.error('Failed to create PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Benchmark results have been saved to artifacts instead.');
             }
-            
-            // Add comparison link
-            comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
-            comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
-            
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: comment
-            });
 
   # Deploy benchmark results to GitHub Pages
   deploy:

.github/workflows/test.yml

@@ -148,6 +148,7 @@ jobs:
       - name: Create test report comment
         if: github.event_name == 'pull_request' && always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
             const fs = require('fs');
@@ -161,34 +162,40 @@ jobs:
               console.error('Error reading test summary:', error);
             }
             
-            // Find existing comment
-            const { data: comments } = await github.rest.issues.listComments({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-            });
-            
-            const botComment = comments.find(comment => 
-              comment.user.type === 'Bot' && 
-              comment.body.includes('## Test Results')
-            );
-            
-            if (botComment) {
-              // Update existing comment
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: botComment.id,
-                body: summary
-              });
-            } else {
-              // Create new comment
-              await github.rest.issues.createComment({
+            try {
+              // Find existing comment
+              const { data: comments } = await github.rest.issues.listComments({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 issue_number: context.issue.number,
-                body: summary
               });
+              
+              const botComment = comments.find(comment => 
+                comment.user.type === 'Bot' && 
+                comment.body.includes('## Test Results')
+              );
+              
+              if (botComment) {
+                // Update existing comment
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: botComment.id,
+                  body: summary
+                });
+              } else {
+                // Create new comment
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: summary
+                });
+              }
+            } catch (error) {
+              console.error('Failed to create/update PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Test results have been saved to the job summary instead.');
             }
       
       # Generate job summary
@@ -260,11 +267,13 @@ jobs:
       - name: Publish test results
         uses: dorny/test-reporter@v1
         if: always()
+        continue-on-error: true
         with:
           name: Test Results
           path: 'artifacts/test-results-*/test-results/junit.xml'
           reporter: java-junit
           fail-on-error: false
+          fail-on-empty: false
       
       # Create a combined artifact with all results
       - name: Create combined results artifact

test-ci-fix.md

@@ -0,0 +1,15 @@
+# Test CI Permission Fixes
+
+This file is created to test the CI permission fixes.
+
+## What was fixed:
+1. Added `continue-on-error: true` to steps that might fail due to permissions
+2. Wrapped GitHub API calls in try-catch blocks
+3. Made benchmark auto-push conditional on main branch pushes only
+
+## Expected behavior:
+- Tests should pass even if PR comments can't be created
+- Benchmarks should run but not fail if they can't push to gh-pages
+- All CI checks should show green when the actual tests pass
+
+This file can be deleted after testing.