mirror of
https://github.com/AutoMaker-Org/automaker.git
synced 2026-03-19 22:53:08 +00:00
Fix agent output validation to prevent false verified status (#807)
* Changes from fix/cursor-fix * feat: Enhance provider error messages with diagnostic context, address test failure, fix port change, move playwright tests to different port * Update apps/ui/src/components/views/board-view/dialogs/add-feature-dialog.tsx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * ci: Update test server port from 3008 to 3108 and add environment configuration * fix: Correct typo in health endpoint URL and standardize port env vars --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { GeminiProvider } from '@/providers/gemini-provider.js';
|
||||
import type { ProviderMessage } from '@automaker/types';
|
||||
|
||||
describe('gemini-provider.ts', () => {
|
||||
let provider: GeminiProvider;
|
||||
@@ -116,4 +117,140 @@ describe('gemini-provider.ts', () => {
|
||||
expect(args[modelIndex + 1]).toBe('gemini-2.5-pro');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeEvent - error handling', () => {
|
||||
it('returns error from result event when status=error and error field is set', () => {
|
||||
const event = {
|
||||
type: 'result',
|
||||
status: 'error',
|
||||
error: 'Model overloaded',
|
||||
session_id: 'sess-gemini-1',
|
||||
stats: { duration_ms: 4000, total_tokens: 0 },
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
expect(msg.error).toBe('Model overloaded');
|
||||
expect(msg.session_id).toBe('sess-gemini-1');
|
||||
});
|
||||
|
||||
it('builds diagnostic fallback when result event has status=error but empty error field', () => {
|
||||
const event = {
|
||||
type: 'result',
|
||||
status: 'error',
|
||||
error: '',
|
||||
session_id: 'sess-gemini-2',
|
||||
stats: { duration_ms: 7500, total_tokens: 0 },
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
// Diagnostic info should be present instead of 'Unknown error'
|
||||
expect(msg.error).toContain('7500ms');
|
||||
expect(msg.error).toContain('sess-gemini-2');
|
||||
expect(msg.error).not.toBe('Unknown error');
|
||||
});
|
||||
|
||||
it('builds fallback with "unknown" duration when stats are missing', () => {
|
||||
const event = {
|
||||
type: 'result',
|
||||
status: 'error',
|
||||
error: '',
|
||||
session_id: 'sess-gemini-nostats',
|
||||
// no stats field
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
expect(msg.error).toContain('unknown');
|
||||
});
|
||||
|
||||
it('returns error from standalone error event with error field set', () => {
|
||||
const event = {
|
||||
type: 'error',
|
||||
error: 'API key invalid',
|
||||
session_id: 'sess-gemini-3',
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
expect(msg.error).toBe('API key invalid');
|
||||
});
|
||||
|
||||
it('builds diagnostic fallback when standalone error event has empty error field', () => {
|
||||
const event = {
|
||||
type: 'error',
|
||||
error: '',
|
||||
session_id: 'sess-gemini-empty',
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
// Should include session_id, not just 'Unknown error'
|
||||
expect(msg.error).toContain('sess-gemini-empty');
|
||||
expect(msg.error).not.toBe('Unknown error');
|
||||
});
|
||||
|
||||
it('builds fallback mentioning "none" when session_id is missing from error event', () => {
|
||||
const event = {
|
||||
type: 'error',
|
||||
error: '',
|
||||
// no session_id
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('error');
|
||||
expect(msg.error).toContain('none');
|
||||
});
|
||||
|
||||
it('uses consistent "Gemini agent failed" label for both result and error event fallbacks', () => {
|
||||
const resultEvent = {
|
||||
type: 'result',
|
||||
status: 'error',
|
||||
error: '',
|
||||
session_id: 'sess-r',
|
||||
stats: { duration_ms: 1000 },
|
||||
};
|
||||
const errorEvent = {
|
||||
type: 'error',
|
||||
error: '',
|
||||
session_id: 'sess-e',
|
||||
};
|
||||
|
||||
const resultMsg = provider.normalizeEvent(resultEvent) as ProviderMessage;
|
||||
const errorMsg = provider.normalizeEvent(errorEvent) as ProviderMessage;
|
||||
|
||||
// Both fallback messages should use the same "Gemini agent failed" prefix
|
||||
expect(resultMsg.error).toContain('Gemini agent failed');
|
||||
expect(errorMsg.error).toContain('Gemini agent failed');
|
||||
});
|
||||
|
||||
it('returns success result when result event has status=success', () => {
|
||||
const event = {
|
||||
type: 'result',
|
||||
status: 'success',
|
||||
error: '',
|
||||
session_id: 'sess-gemini-ok',
|
||||
stats: { duration_ms: 1200, total_tokens: 500 },
|
||||
};
|
||||
|
||||
const msg = provider.normalizeEvent(event) as ProviderMessage;
|
||||
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg.type).toBe('result');
|
||||
expect(msg.subtype).toBe('success');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user