Fix agent output validation to prevent false verified status (#807)

* Changes from fix/cursor-fix

* feat: Enhance provider error messages with diagnostic context, address test failure, fix port change, move playwright tests to different port

* Update apps/ui/src/components/views/board-view/dialogs/add-feature-dialog.tsx

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* ci: Update test server port from 3008 to 3108 and add environment configuration

* fix: Correct typo in health endpoint URL and standardize port env vars

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
gsxdsm
2026-02-24 20:18:40 -08:00
committed by GitHub
parent 0330c70261
commit 51e9a23ba1
36 changed files with 1610 additions and 104 deletions

View File

@@ -1,5 +1,6 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { GeminiProvider } from '@/providers/gemini-provider.js';
import type { ProviderMessage } from '@automaker/types';
describe('gemini-provider.ts', () => {
let provider: GeminiProvider;
@@ -116,4 +117,140 @@ describe('gemini-provider.ts', () => {
expect(args[modelIndex + 1]).toBe('gemini-2.5-pro');
});
});
describe('normalizeEvent - error handling', () => {
it('returns error from result event when status=error and error field is set', () => {
const event = {
type: 'result',
status: 'error',
error: 'Model overloaded',
session_id: 'sess-gemini-1',
stats: { duration_ms: 4000, total_tokens: 0 },
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
expect(msg.error).toBe('Model overloaded');
expect(msg.session_id).toBe('sess-gemini-1');
});
it('builds diagnostic fallback when result event has status=error but empty error field', () => {
const event = {
type: 'result',
status: 'error',
error: '',
session_id: 'sess-gemini-2',
stats: { duration_ms: 7500, total_tokens: 0 },
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
// Diagnostic info should be present instead of 'Unknown error'
expect(msg.error).toContain('7500ms');
expect(msg.error).toContain('sess-gemini-2');
expect(msg.error).not.toBe('Unknown error');
});
it('builds fallback with "unknown" duration when stats are missing', () => {
const event = {
type: 'result',
status: 'error',
error: '',
session_id: 'sess-gemini-nostats',
// no stats field
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
expect(msg.error).toContain('unknown');
});
it('returns error from standalone error event with error field set', () => {
const event = {
type: 'error',
error: 'API key invalid',
session_id: 'sess-gemini-3',
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
expect(msg.error).toBe('API key invalid');
});
it('builds diagnostic fallback when standalone error event has empty error field', () => {
const event = {
type: 'error',
error: '',
session_id: 'sess-gemini-empty',
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
// Should include session_id, not just 'Unknown error'
expect(msg.error).toContain('sess-gemini-empty');
expect(msg.error).not.toBe('Unknown error');
});
it('builds fallback mentioning "none" when session_id is missing from error event', () => {
const event = {
type: 'error',
error: '',
// no session_id
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('error');
expect(msg.error).toContain('none');
});
it('uses consistent "Gemini agent failed" label for both result and error event fallbacks', () => {
const resultEvent = {
type: 'result',
status: 'error',
error: '',
session_id: 'sess-r',
stats: { duration_ms: 1000 },
};
const errorEvent = {
type: 'error',
error: '',
session_id: 'sess-e',
};
const resultMsg = provider.normalizeEvent(resultEvent) as ProviderMessage;
const errorMsg = provider.normalizeEvent(errorEvent) as ProviderMessage;
// Both fallback messages should use the same "Gemini agent failed" prefix
expect(resultMsg.error).toContain('Gemini agent failed');
expect(errorMsg.error).toContain('Gemini agent failed');
});
it('returns success result when result event has status=success', () => {
const event = {
type: 'result',
status: 'success',
error: '',
session_id: 'sess-gemini-ok',
stats: { duration_ms: 1200, total_tokens: 500 },
};
const msg = provider.normalizeEvent(event) as ProviderMessage;
expect(msg).not.toBeNull();
expect(msg.type).toBe('result');
expect(msg.subtype).toBe('success');
});
});
});