feat: complete Phase 3.8 - test infrastructure and CI/CD enhancements

- Add test result artifacts storage with multiple formats (JUnit, JSON, HTML) - Configure GitHub Actions to upload and preserve test outputs - Add PR comment integration with test summaries - Create benchmark comparison workflow for PR performance tracking - Add detailed test report generation scripts - Configure artifact retention policies (30 days for tests, 90 for combined) - Set up test metadata collection for better debugging This completes all remaining test infrastructure tasks and provides comprehensive visibility into test results across CI/CD pipeline.
2025-07-28 22:56:15 +02:00
parent b5210e5963
commit 61de107c4b
9 changed files with 1680 additions and 12 deletions
--- a/.github/workflows/benchmark-pr.yml
+++ b/.github/workflows/benchmark-pr.yml
@@ -0,0 +1,154 @@
+name: Benchmark PR Comparison
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'src/**'
+      - 'tests/benchmarks/**'
+      - 'package.json'
+      - 'vitest.config.benchmark.ts'
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  benchmark-comparison:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      # Run benchmarks on current branch
+      - name: Run current benchmarks
+        run: npm run benchmark:ci
+      
+      - name: Save current results
+        run: cp benchmark-results.json benchmark-current.json
+      
+      # Checkout and run benchmarks on base branch
+      - name: Checkout base branch
+        run: |
+          git checkout ${{ github.event.pull_request.base.sha }}
+          git status
+      
+      - name: Install base dependencies
+        run: npm ci
+      
+      - name: Run baseline benchmarks
+        run: npm run benchmark:ci
+        continue-on-error: true
+      
+      - name: Save baseline results
+        run: |
+          if [ -f benchmark-results.json ]; then
+            cp benchmark-results.json benchmark-baseline.json
+          else
+            echo '{"files":[]}' > benchmark-baseline.json
+          fi
+      
+      # Compare results
+      - name: Checkout PR branch again
+        run: git checkout ${{ github.event.pull_request.head.sha }}
+      
+      - name: Compare benchmarks
+        id: compare
+        run: |
+          node scripts/compare-benchmarks.js benchmark-current.json benchmark-baseline.json || echo "REGRESSION=true" >> $GITHUB_OUTPUT
+      
+      # Upload comparison artifacts
+      - name: Upload benchmark comparison
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-comparison-${{ github.run_number }}
+          path: |
+            benchmark-current.json
+            benchmark-baseline.json
+            benchmark-comparison.json
+            benchmark-comparison.md
+          retention-days: 30
+      
+      # Post comparison to PR
+      - name: Post benchmark comparison to PR
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let comment = '## ⚡ Benchmark Comparison\n\n';
+            
+            try {
+              if (fs.existsSync('benchmark-comparison.md')) {
+                const comparison = fs.readFileSync('benchmark-comparison.md', 'utf8');
+                comment += comparison;
+              } else {
+                comment += 'Benchmark comparison could not be generated.';
+              }
+            } catch (error) {
+              comment += `Error reading benchmark comparison: ${error.message}`;
+            }
+            
+            comment += '\n\n---\n';
+            comment += `*[View full benchmark results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*`;
+            
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            
+            const botComment = comments.find(comment => 
+              comment.user.type === 'Bot' && 
+              comment.body.includes('## ⚡ Benchmark Comparison')
+            );
+            
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: comment
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: comment
+              });
+            }
+      
+      # Add status check
+      - name: Set benchmark status
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const hasRegression = '${{ steps.compare.outputs.REGRESSION }}' === 'true';
+            const state = hasRegression ? 'failure' : 'success';
+            const description = hasRegression 
+              ? 'Performance regressions detected' 
+              : 'No performance regressions';
+            
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: context.sha,
+              state: state,
+              target_url: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: description,
+              context: 'benchmarks/regression-check'
+            });