feat: enhance AI validation with PR analysis and UI improvements

- Replace HTML checkbox with proper UI Checkbox component - Add system prompt instructions for AI to check PR changes via gh CLI - Add PRAnalysis schema field with recommendation (wait_for_merge, pr_needs_work, no_pr) - Show detailed PR analysis badge in validation dialog - Hide "Convert to Task" button when PR fix is ready (wait_for_merge) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-30 06:12:03 +00:00 · 2025-12-28 22:22:14 +01:00
parent 96196f906f
commit 97ae4b6362
5 changed files with 156 additions and 25 deletions
--- a/apps/server/src/routes/github/routes/validation-schema.ts
+++ b/apps/server/src/routes/github/routes/validation-schema.ts
@@ -49,6 +49,34 @@ export const issueValidationSchema = {
      enum: ['trivial', 'simple', 'moderate', 'complex', 'very_complex'],
      description: 'Estimated effort to address the issue',
    },
+    prAnalysis: {
+      type: 'object',
+      properties: {
+        hasOpenPR: {
+          type: 'boolean',
+          description: 'Whether there is an open PR linked to this issue',
+        },
+        prFixesIssue: {
+          type: 'boolean',
+          description: 'Whether the PR appears to fix the issue based on the diff',
+        },
+        prNumber: {
+          type: 'number',
+          description: 'The PR number that was analyzed',
+        },
+        prSummary: {
+          type: 'string',
+          description: 'Brief summary of what the PR changes',
+        },
+        recommendation: {
+          type: 'string',
+          enum: ['wait_for_merge', 'pr_needs_work', 'no_pr'],
+          description:
+            'Recommendation: wait for PR to merge, PR needs more work, or no relevant PR',
+        },
+      },
+      description: 'Analysis of linked pull requests if any exist',
+    },
  },
  required: ['verdict', 'confidence', 'reasoning'],
  additionalProperties: false,
@@ -67,7 +95,8 @@ Your task is to analyze a GitHub issue and determine if it's valid by scanning t
 1. **Read the issue carefully** - Understand what is being reported or requested
 2. **Search the codebase** - Use Glob to find relevant files by pattern, Grep to search for keywords
 3. **Examine the code** - Use Read to look at the actual implementation in relevant files
-4. **Form your verdict** - Based on your analysis, determine if the issue is valid
+4. **Check linked PRs** - If there are linked pull requests, use \`gh pr diff <PR_NUMBER>\` to review the changes
+5. **Form your verdict** - Based on your analysis, determine if the issue is valid

 ## Verdicts

@@ -88,12 +117,32 @@ Your task is to analyze a GitHub issue and determine if it's valid by scanning t
 - Is the implementation location clear?
 - Is the request technically feasible given the codebase structure?

+## Analyzing Linked Pull Requests
+
+When an issue has linked PRs (especially open ones), you MUST analyze them:
+
+1. **Run \`gh pr diff <PR_NUMBER>\`** to see what changes the PR makes
+2. **Run \`gh pr view <PR_NUMBER>\`** to see PR description and status
+3. **Evaluate if the PR fixes the issue** - Does the diff address the reported problem?
+4. **Provide a recommendation**:
+   - \`wait_for_merge\`: The PR appears to fix the issue correctly. No additional work needed - just wait for it to be merged.
+   - \`pr_needs_work\`: The PR attempts to fix the issue but is incomplete or has problems.
+   - \`no_pr\`: No relevant PR exists for this issue.
+
+5. **Include prAnalysis in your response** with:
+   - hasOpenPR: true/false
+   - prFixesIssue: true/false (based on diff analysis)
+   - prNumber: the PR number you analyzed
+   - prSummary: brief description of what the PR changes
+   - recommendation: one of the above values
+
 ## Response Guidelines

 - **Always include relatedFiles** when you find relevant code
 - **Set bugConfirmed to true** only if you can definitively confirm a bug exists in the code
 - **Provide a suggestedFix** when you have a clear idea of how to address the issue
 - **Use missingInfo** when the verdict is needs_clarification to list what's needed
+- **Include prAnalysis** when there are linked PRs - this is critical for avoiding duplicate work
 - **Set estimatedComplexity** to help prioritize:
  - trivial: Simple text changes, one-line fixes
  - simple: Small changes to one file
--- a/apps/ui/src/components/views/github-issues-view/components/issue-detail-panel.tsx
+++ b/apps/ui/src/components/views/github-issues-view/components/issue-detail-panel.tsx
@@ -16,6 +16,7 @@ import {
 } from 'lucide-react';
 import { useState } from 'react';
 import { Button } from '@/components/ui/button';
+import { Checkbox } from '@/components/ui/checkbox';
 import { Markdown } from '@/components/ui/markdown';
 import { cn } from '@/lib/utils';
 import type { IssueDetailPanelProps } from '../types';
@@ -291,12 +292,10 @@ export function IssueDetailPanel({
              )}
            </button>
            {comments.length > 0 && (
-              <label className="flex items-center gap-1.5 text-xs text-muted-foreground cursor-pointer">
-                <input
-                  type="checkbox"
+              <label className="flex items-center gap-2 text-xs text-muted-foreground cursor-pointer">
+                <Checkbox
                  checked={includeCommentsInAnalysis}
-                  onChange={(e) => setIncludeCommentsInAnalysis(e.target.checked)}
-                  className="h-3 w-3 rounded border-border"
+                  onCheckedChange={setIncludeCommentsInAnalysis}
                />
                Include in AI analysis
              </label>
--- a/apps/ui/src/components/views/github-issues-view/dialogs/validation-dialog.tsx
+++ b/apps/ui/src/components/views/github-issues-view/dialogs/validation-dialog.tsx
@@ -17,6 +17,8 @@ import {
  AlertTriangle,
  Plus,
  GitPullRequest,
+  Clock,
+  Wrench,
 } from 'lucide-react';
 import { cn } from '@/lib/utils';
 import type {
@@ -150,23 +152,77 @@ export function ValidationDialog({
              </div>
            )}

-            {/* Work in Progress Badge - Show when there's an open PR linked */}
-            {issue.linkedPRs?.some((pr) => pr.state === 'open' || pr.state === 'OPEN') && (
-              <div className="flex items-center gap-2 p-3 rounded-lg bg-purple-500/10 border border-purple-500/20">
-                <GitPullRequest className="h-5 w-5 text-purple-500 shrink-0" />
-                <div className="flex-1">
-                  <span className="text-sm font-medium text-purple-500">Work in Progress</span>
-                  <p className="text-xs text-muted-foreground mt-0.5">
-                    {issue.linkedPRs
-                      .filter((pr) => pr.state === 'open' || pr.state === 'OPEN')
-                      .map((pr) => `PR #${pr.number}`)
-                      .join(', ')}{' '}
-                    is open for this issue
-                  </p>
+            {/* PR Analysis Section - Show AI's analysis of linked PRs */}
+            {validationResult.prAnalysis && validationResult.prAnalysis.hasOpenPR && (
+              <div
+                className={cn(
+                  'p-3 rounded-lg border',
+                  validationResult.prAnalysis.recommendation === 'wait_for_merge'
+                    ? 'bg-green-500/10 border-green-500/20'
+                    : validationResult.prAnalysis.recommendation === 'pr_needs_work'
+                      ? 'bg-yellow-500/10 border-yellow-500/20'
+                      : 'bg-purple-500/10 border-purple-500/20'
+                )}
+              >
+                <div className="flex items-start gap-2">
+                  {validationResult.prAnalysis.recommendation === 'wait_for_merge' ? (
+                    <Clock className="h-5 w-5 text-green-500 shrink-0 mt-0.5" />
+                  ) : validationResult.prAnalysis.recommendation === 'pr_needs_work' ? (
+                    <Wrench className="h-5 w-5 text-yellow-500 shrink-0 mt-0.5" />
+                  ) : (
+                    <GitPullRequest className="h-5 w-5 text-purple-500 shrink-0 mt-0.5" />
+                  )}
+                  <div className="flex-1">
+                    <span
+                      className={cn(
+                        'text-sm font-medium',
+                        validationResult.prAnalysis.recommendation === 'wait_for_merge'
+                          ? 'text-green-500'
+                          : validationResult.prAnalysis.recommendation === 'pr_needs_work'
+                            ? 'text-yellow-500'
+                            : 'text-purple-500'
+                      )}
+                    >
+                      {validationResult.prAnalysis.recommendation === 'wait_for_merge'
+                        ? 'Fix Ready - Wait for Merge'
+                        : validationResult.prAnalysis.recommendation === 'pr_needs_work'
+                          ? 'PR Needs Work'
+                          : 'Work in Progress'}
+                    </span>
+                    {validationResult.prAnalysis.prNumber && (
+                      <p className="text-xs text-muted-foreground mt-0.5">
+                        PR #{validationResult.prAnalysis.prNumber}
+                        {validationResult.prAnalysis.prFixesIssue && ' appears to fix this issue'}
+                      </p>
+                    )}
+                    {validationResult.prAnalysis.prSummary && (
+                      <p className="text-xs text-muted-foreground mt-1">
+                        {validationResult.prAnalysis.prSummary}
+                      </p>
+                    )}
+                  </div>
                </div>
              </div>
            )}

+            {/* Fallback Work in Progress Badge - Show when there's an open PR but no AI analysis */}
+            {!validationResult.prAnalysis?.hasOpenPR &&
+              issue.linkedPRs?.some((pr) => pr.state === 'open' || pr.state === 'OPEN') && (
+                <div className="flex items-center gap-2 p-3 rounded-lg bg-purple-500/10 border border-purple-500/20">
+                  <GitPullRequest className="h-5 w-5 text-purple-500 shrink-0" />
+                  <div className="flex-1">
+                    <span className="text-sm font-medium text-purple-500">Work in Progress</span>
+                    <p className="text-xs text-muted-foreground mt-0.5">
+                      {issue.linkedPRs
+                        .filter((pr) => pr.state === 'open' || pr.state === 'OPEN')
+                        .map((pr) => `PR #${pr.number}`)
+                        .join(', ')}{' '}
+                      is open for this issue
+                    </p>
+                  </div>
+                </div>
+              )}
+
            {/* Reasoning */}
            <div className="space-y-2">
              <h4 className="text-sm font-medium flex items-center gap-2">
@@ -236,12 +292,14 @@ export function ValidationDialog({
          <Button variant="ghost" onClick={() => onOpenChange(false)}>
            Close
          </Button>
-          {validationResult?.verdict === 'valid' && onConvertToTask && (
-            <Button onClick={handleConvertToTask}>
-              <Plus className="h-4 w-4 mr-2" />
-              Convert to Task
-            </Button>
-          )}
+          {validationResult?.verdict === 'valid' &&
+            onConvertToTask &&
+            validationResult?.prAnalysis?.recommendation !== 'wait_for_merge' && (
+              <Button onClick={handleConvertToTask}>
+                <Plus className="h-4 w-4 mr-2" />
+                Convert to Task
+              </Button>
+            )}
        </DialogFooter>
      </DialogContent>
    </Dialog>
--- a/libs/types/src/index.ts
+++ b/libs/types/src/index.ts
@@ -87,6 +87,8 @@ export type {
  IssueValidationVerdict,
  IssueValidationConfidence,
  IssueComplexity,
+  PRRecommendation,
+  PRAnalysis,
  LinkedPRInfo,
  IssueValidationInput,
  IssueValidationRequest,
--- a/libs/types/src/issue-validation.ts
+++ b/libs/types/src/issue-validation.ts
@@ -21,6 +21,27 @@ export type IssueValidationConfidence = 'high' | 'medium' | 'low';
 */
 export type IssueComplexity = 'trivial' | 'simple' | 'moderate' | 'complex' | 'very_complex';

+/**
+ * Recommendation for PR-related action
+ */
+export type PRRecommendation = 'wait_for_merge' | 'pr_needs_work' | 'no_pr';
+
+/**
+ * Analysis of a linked pull request
+ */
+export interface PRAnalysis {
+  /** Whether there is an open PR linked to this issue */
+  hasOpenPR: boolean;
+  /** Whether the PR appears to fix the issue based on the diff */
+  prFixesIssue?: boolean;
+  /** The PR number that was analyzed */
+  prNumber?: number;
+  /** Brief summary of what the PR changes */
+  prSummary?: string;
+  /** Recommendation: wait for PR to merge, PR needs more work, or no relevant PR */
+  recommendation: PRRecommendation;
+}
+
 /**
 * Linked PR info for validation
 */
@@ -73,6 +94,8 @@ export interface IssueValidationResult {
  missingInfo?: string[];
  /** Estimated effort to address the issue */
  estimatedComplexity?: IssueComplexity;
+  /** Analysis of linked pull requests (if any) */
+  prAnalysis?: PRAnalysis;
 }

 /**