Fix agent output validation to prevent false verified status (#807)

* Changes from fix/cursor-fix

* feat: Enhance provider error messages with diagnostic context, address test failure, fix port change, move playwright tests to different port

* Update apps/ui/src/components/views/board-view/dialogs/add-feature-dialog.tsx

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* ci: Update test server port from 3008 to 3108 and add environment configuration

* fix: Correct typo in health endpoint URL and standardize port env vars

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

apps/server/tests/unit/providers/copilot-provider.test.ts

@@ -397,6 +397,45 @@ describe('copilot-provider.ts', () => {
       });
     });
 
+    it('should use error code in fallback when session.error message is empty', () => {
+      const event = {
+        type: 'session.error',
+        data: { message: '', code: 'RATE_LIMIT_EXCEEDED' },
+      };
+
+      const result = provider.normalizeEvent(event);
+      expect(result).not.toBeNull();
+      expect(result!.type).toBe('error');
+      expect(result!.error).toContain('RATE_LIMIT_EXCEEDED');
+      expect(result!.error).not.toBe('Unknown error');
+    });
+
+    it('should return generic "Copilot agent error" fallback when both message and code are empty', () => {
+      const event = {
+        type: 'session.error',
+        data: { message: '', code: '' },
+      };
+
+      const result = provider.normalizeEvent(event);
+      expect(result).not.toBeNull();
+      expect(result!.type).toBe('error');
+      expect(result!.error).toBe('Copilot agent error');
+      // Must NOT be the old opaque 'Unknown error'
+      expect(result!.error).not.toBe('Unknown error');
+    });
+
+    it('should return generic "Copilot agent error" fallback when data has no code field', () => {
+      const event = {
+        type: 'session.error',
+        data: { message: '' },
+      };
+
+      const result = provider.normalizeEvent(event);
+      expect(result).not.toBeNull();
+      expect(result!.type).toBe('error');
+      expect(result!.error).toBe('Copilot agent error');
+    });
+
     it('should return null for unknown event types', () => {
       const event = { type: 'unknown.event' };
 

apps/server/tests/unit/providers/cursor-provider.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, beforeEach } from 'vitest';
 import { CursorProvider } from '@/providers/cursor-provider.js';
 
 describe('cursor-provider.ts', () => {
@@ -36,4 +36,122 @@ describe('cursor-provider.ts', () => {
       expect(args).not.toContain('--resume');
     });
   });
+
+  describe('normalizeEvent - result error handling', () => {
+    let provider: CursorProvider;
+
+    beforeEach(() => {
+      provider = Object.create(CursorProvider.prototype) as CursorProvider;
+    });
+
+    it('returns error message from resultEvent.error when is_error=true', () => {
+      const event = {
+        type: 'result',
+        is_error: true,
+        error: 'Rate limit exceeded',
+        result: '',
+        subtype: 'error',
+        duration_ms: 3000,
+        session_id: 'sess-123',
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg).not.toBeNull();
+      expect(msg!.type).toBe('error');
+      expect(msg!.error).toBe('Rate limit exceeded');
+    });
+
+    it('falls back to resultEvent.result when error field is empty and is_error=true', () => {
+      const event = {
+        type: 'result',
+        is_error: true,
+        error: '',
+        result: 'Process terminated unexpectedly',
+        subtype: 'error',
+        duration_ms: 5000,
+        session_id: 'sess-456',
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg).not.toBeNull();
+      expect(msg!.type).toBe('error');
+      expect(msg!.error).toBe('Process terminated unexpectedly');
+    });
+
+    it('builds diagnostic fallback when both error and result are empty and is_error=true', () => {
+      const event = {
+        type: 'result',
+        is_error: true,
+        error: '',
+        result: '',
+        subtype: 'error',
+        duration_ms: 5000,
+        session_id: 'sess-789',
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg).not.toBeNull();
+      expect(msg!.type).toBe('error');
+      // Should contain diagnostic info rather than 'Unknown error'
+      expect(msg!.error).toContain('5000ms');
+      expect(msg!.error).toContain('sess-789');
+      expect(msg!.error).not.toBe('Unknown error');
+    });
+
+    it('preserves session_id in error message', () => {
+      const event = {
+        type: 'result',
+        is_error: true,
+        error: 'Timeout occurred',
+        result: '',
+        subtype: 'error',
+        duration_ms: 30000,
+        session_id: 'my-session-id',
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg!.session_id).toBe('my-session-id');
+    });
+
+    it('uses "none" when session_id is missing from diagnostic fallback', () => {
+      const event = {
+        type: 'result',
+        is_error: true,
+        error: '',
+        result: '',
+        subtype: 'error',
+        duration_ms: 5000,
+        // session_id intentionally omitted
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg).not.toBeNull();
+      expect(msg!.type).toBe('error');
+      expect(msg!.error).toContain('none');
+      expect(msg!.error).not.toContain('undefined');
+    });
+
+    it('returns success result when is_error=false', () => {
+      const event = {
+        type: 'result',
+        is_error: false,
+        error: '',
+        result: 'Completed successfully',
+        subtype: 'success',
+        duration_ms: 2000,
+        session_id: 'sess-ok',
+      };
+
+      const msg = provider.normalizeEvent(event);
+
+      expect(msg).not.toBeNull();
+      expect(msg!.type).toBe('result');
+      expect(msg!.subtype).toBe('success');
+    });
+  });
 });

apps/server/tests/unit/providers/gemini-provider.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect, beforeEach } from 'vitest';
 import { GeminiProvider } from '@/providers/gemini-provider.js';
+import type { ProviderMessage } from '@automaker/types';
 
 describe('gemini-provider.ts', () => {
   let provider: GeminiProvider;
@@ -116,4 +117,140 @@ describe('gemini-provider.ts', () => {
       expect(args[modelIndex + 1]).toBe('gemini-2.5-pro');
     });
   });
+
+  describe('normalizeEvent - error handling', () => {
+    it('returns error from result event when status=error and error field is set', () => {
+      const event = {
+        type: 'result',
+        status: 'error',
+        error: 'Model overloaded',
+        session_id: 'sess-gemini-1',
+        stats: { duration_ms: 4000, total_tokens: 0 },
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      expect(msg.error).toBe('Model overloaded');
+      expect(msg.session_id).toBe('sess-gemini-1');
+    });
+
+    it('builds diagnostic fallback when result event has status=error but empty error field', () => {
+      const event = {
+        type: 'result',
+        status: 'error',
+        error: '',
+        session_id: 'sess-gemini-2',
+        stats: { duration_ms: 7500, total_tokens: 0 },
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      // Diagnostic info should be present instead of 'Unknown error'
+      expect(msg.error).toContain('7500ms');
+      expect(msg.error).toContain('sess-gemini-2');
+      expect(msg.error).not.toBe('Unknown error');
+    });
+
+    it('builds fallback with "unknown" duration when stats are missing', () => {
+      const event = {
+        type: 'result',
+        status: 'error',
+        error: '',
+        session_id: 'sess-gemini-nostats',
+        // no stats field
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      expect(msg.error).toContain('unknown');
+    });
+
+    it('returns error from standalone error event with error field set', () => {
+      const event = {
+        type: 'error',
+        error: 'API key invalid',
+        session_id: 'sess-gemini-3',
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      expect(msg.error).toBe('API key invalid');
+    });
+
+    it('builds diagnostic fallback when standalone error event has empty error field', () => {
+      const event = {
+        type: 'error',
+        error: '',
+        session_id: 'sess-gemini-empty',
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      // Should include session_id, not just 'Unknown error'
+      expect(msg.error).toContain('sess-gemini-empty');
+      expect(msg.error).not.toBe('Unknown error');
+    });
+
+    it('builds fallback mentioning "none" when session_id is missing from error event', () => {
+      const event = {
+        type: 'error',
+        error: '',
+        // no session_id
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('error');
+      expect(msg.error).toContain('none');
+    });
+
+    it('uses consistent "Gemini agent failed" label for both result and error event fallbacks', () => {
+      const resultEvent = {
+        type: 'result',
+        status: 'error',
+        error: '',
+        session_id: 'sess-r',
+        stats: { duration_ms: 1000 },
+      };
+      const errorEvent = {
+        type: 'error',
+        error: '',
+        session_id: 'sess-e',
+      };
+
+      const resultMsg = provider.normalizeEvent(resultEvent) as ProviderMessage;
+      const errorMsg = provider.normalizeEvent(errorEvent) as ProviderMessage;
+
+      // Both fallback messages should use the same "Gemini agent failed" prefix
+      expect(resultMsg.error).toContain('Gemini agent failed');
+      expect(errorMsg.error).toContain('Gemini agent failed');
+    });
+
+    it('returns success result when result event has status=success', () => {
+      const event = {
+        type: 'result',
+        status: 'success',
+        error: '',
+        session_id: 'sess-gemini-ok',
+        stats: { duration_ms: 1200, total_tokens: 500 },
+      };
+
+      const msg = provider.normalizeEvent(event) as ProviderMessage;
+
+      expect(msg).not.toBeNull();
+      expect(msg.type).toBe('result');
+      expect(msg.subtype).toBe('success');
+    });
+  });
 });