Merge pull request #127 from czlonkowski/test/ci-permission-fixes

fix: handle GitHub Actions permission errors gracefully

.github/workflows/benchmark-pr.yml

@@ -93,71 +93,84 @@ jobs:
       - name: Post benchmark comparison to PR
         if: always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
-            const fs = require('fs');
-            let comment = '## ⚡ Benchmark Comparison\n\n';
-            
             try {
-              if (fs.existsSync('benchmark-comparison.md')) {
+              const fs = require('fs');
-                const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
+              let comment = '## ⚡ Benchmark Comparison\n\n';
-                comment += comparison;
+              
-              } else {
+              try {
-                comment += 'Benchmark comparison could not be generated.';
+                if (fs.existsSync('benchmark-comparison.md')) {
+                  const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
+                  comment += comparison;
+                } else {
+                  comment += 'Benchmark comparison could not be generated.';
+                }
+              } catch (error) {
+                comment += `Error reading benchmark comparison: ${error.message}`;
               }
-            } catch (error) {
-              comment += `Error reading benchmark comparison: ${error.message}`;
-            }
               
-            comment += '\n\n---\n';
+              comment += '\n\n---\n';
-            comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
+              comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
               
-            // Find existing comment
+              // Find existing comment
-            const { data: comments } = await github.rest.issues.listComments({
+              const { data: comments } = await github.rest.issues.listComments({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-            });
-            
-            const botComment = comments.find(comment => 
-              comment.user.type === 'Bot' && 
-              comment.body.includes('## ⚡ Benchmark Comparison')
-            );
-            
-            if (botComment) {
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: botComment.id,
-                body: comment
-              });
-            } else {
-              await github.rest.issues.createComment({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 issue_number: context.issue.number,
-                body: comment
               });
+              
+              const botComment = comments.find(comment => 
+                comment.user.type === 'Bot' && 
+                comment.body.includes('## ⚡ Benchmark Comparison')
+              );
+              
+              if (botComment) {
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: botComment.id,
+                  body: comment
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: comment
+                });
+              }
+            } catch (error) {
+              console.error('Failed to create/update PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Benchmark comparison has been saved to artifacts instead.');
             }
       
       # Add status check
       - name: Set benchmark status
         if: always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
-            const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
+            try {
-            const state = hasRegression ? 'failure' : 'success';
+              const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
-            const description = hasRegression 
+              const state = hasRegression ? 'failure' : 'success';
-              ? 'Performance regressions detected' 
+              const description = hasRegression 
-              : 'No performance regressions';
+                ? 'Performance regressions detected' 
+                : 'No performance regressions';
               
-            await github.rest.repos.createCommitStatus({
+              await github.rest.repos.createCommitStatus({
-              owner: context.repo.owner,
+                owner: context.repo.owner,
-              repo: context.repo.repo,
+                repo: context.repo.repo,
-              sha: context.sha,
+                sha: context.sha,
-              state: state,
+                state: state,
-              target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
-              description: description,
+                description: description,
-              context: 'benchmarks/regression-check'
+                context: 'benchmarks/regression-check'
-            });
+              });
+            } catch (error) {
+              console.error('Failed to create commit status:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+            }

.github/workflows/benchmark.yml

@@ -103,12 +103,14 @@ jobs:
       # Store benchmark results and compare
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
+        continue-on-error: true
+        id: benchmark
         with:
           name: n8n-mcp Benchmarks
           tool: 'customSmallerIsBetter'
           output-file-path: benchmark-results-formatted.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          auto-push: true
+          auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
           # Where to store benchmark data
           benchmark-data-dir-path: 'benchmarks'
           # Alert when performance regresses by 10%
@@ -120,53 +122,61 @@ jobs:
           summary-always: true
           # Max number of data points to retain
           max-items-in-chart: 50
+          fail-on-alert: false
 
       # Comment on PR with benchmark results
       - name: Comment PR with results
         uses: actions/github-script@v7
         if: github.event_name == 'pull_request'
+        continue-on-error: true
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const fs = require('fs');
+            try {
-            const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
+              const fs = require('fs');
+              const summary = JSON.parse(fs.readFileSync('benchmark-summary.json', 'utf8'));
               
-            // Format results for PR comment
+              // Format results for PR comment
-            let comment = '## 📊 Performance Benchmark Results\n\n';
+              let comment = '## 📊 Performance Benchmark Results\n\n';
-            comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
+              comment += `🕐 Run at: ${new Date(summary.timestamp).toLocaleString()}\n\n`;
-            comment += '| Benchmark | Time | Ops/sec | Range |\n';
+              comment += '| Benchmark | Time | Ops/sec | Range |\n';
-            comment += '|-----------|------|---------|-------|\n';
+              comment += '|-----------|------|---------|-------|\n';
               
-            // Group benchmarks by category
+              // Group benchmarks by category
-            const categories = {};
+              const categories = {};
-            for (const benchmark of summary.benchmarks) {
+              for (const benchmark of summary.benchmarks) {
-              const [category, ...nameParts] = benchmark.name.split(' - ');
+                const [category, ...nameParts] = benchmark.name.split(' - ');
-              if (!categories[category]) categories[category] = [];
+                if (!categories[category]) categories[category] = [];
-              categories[category].push({
+                categories[category].push({
-                ...benchmark,
+                  ...benchmark,
-                shortName: nameParts.join(' - ')
+                  shortName: nameParts.join(' - ')
-              });
+                });
-            }
-            
-            // Display by category
-            for (const [category, benchmarks] of Object.entries(categories)) {
-              comment += `\n### ${category}\n`;
-              for (const benchmark of benchmarks) {
-                comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
               }
+              
+              // Display by category
+              for (const [category, benchmarks] of Object.entries(categories)) {
+                comment += `\n### ${category}\n`;
+                for (const benchmark of benchmarks) {
+                  comment += `| ${benchmark.shortName} | ${benchmark.time} | ${benchmark.opsPerSec} | ${benchmark.range} |\n`;
+                }
+              }
+              
+              // Add comparison link
+              comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
+              comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
+              
+              await github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: comment
+              });
+            } catch (error) {
+              console.error('Failed to create PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Benchmark results have been saved to artifacts instead.');
             }
 
-            // Add comparison link
-            comment += '\n\n📈 [View historical benchmark trends](https://czlonkowski.github.io/n8n-mcp/benchmarks/)\n';
-            comment += '\n⚡ Performance regressions >10% will be flagged automatically.\n';
-            
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: comment
-            });
-
   # Deploy benchmark results to GitHub Pages
   deploy:
     needs: benchmark

.github/workflows/test.yml

@@ -148,6 +148,7 @@ jobs:
       - name: Create test report comment
         if: github.event_name == 'pull_request' && always()
         uses: actions/github-script@v7
+        continue-on-error: true
         with:
           script: |
             const fs = require('fs');
@@ -161,34 +162,40 @@ jobs:
               console.error('Error reading test summary:', error);
             }
             
-            // Find existing comment
+            try {
-            const { data: comments } = await github.rest.issues.listComments({
+              // Find existing comment
-              owner: context.repo.owner,
+              const { data: comments } = await github.rest.issues.listComments({
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-            });
-            
-            const botComment = comments.find(comment => 
-              comment.user.type === 'Bot' && 
-              comment.body.includes('## Test Results')
-            );
-            
-            if (botComment) {
-              // Update existing comment
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: botComment.id,
-                body: summary
-              });
-            } else {
-              // Create new comment
-              await github.rest.issues.createComment({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 issue_number: context.issue.number,
-                body: summary
               });
+              
+              const botComment = comments.find(comment => 
+                comment.user.type === 'Bot' && 
+                comment.body.includes('## Test Results')
+              );
+              
+              if (botComment) {
+                // Update existing comment
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: botComment.id,
+                  body: summary
+                });
+              } else {
+                // Create new comment
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: summary
+                });
+              }
+            } catch (error) {
+              console.error('Failed to create/update PR comment:', error.message);
+              console.log('This is likely due to insufficient permissions for external PRs.');
+              console.log('Test results have been saved to the job summary instead.');
             }
       
       # Generate job summary
@@ -260,11 +267,13 @@ jobs:
       - name: Publish test results
         uses: dorny/test-reporter@v1
         if: always()
+        continue-on-error: true
         with:
           name: Test Results
           path: 'artifacts/test-results-*/test-results/junit.xml'
           reporter: java-junit
           fail-on-error: false
+          fail-on-empty: false
       
       # Create a combined artifact with all results
       - name: Create combined results artifact