670 lines
19 KiB
JavaScript
670 lines
19 KiB
JavaScript
import { jest } from '@jest/globals';
|
|
|
|
// Mock the 'ai' SDK
|
|
const mockGenerateText = jest.fn();
|
|
const mockGenerateObject = jest.fn();
|
|
const mockNoObjectGeneratedError = class NoObjectGeneratedError extends Error {
|
|
static isInstance(error) {
|
|
return error instanceof mockNoObjectGeneratedError;
|
|
}
|
|
constructor(cause) {
|
|
super('No object generated');
|
|
this.cause = cause;
|
|
this.usage = cause.usage;
|
|
}
|
|
};
|
|
const mockJSONParseError = class JSONParseError extends Error {
|
|
constructor(text) {
|
|
super('JSON parse error');
|
|
this.text = text;
|
|
}
|
|
};
|
|
|
|
jest.unstable_mockModule('ai', () => ({
|
|
generateText: mockGenerateText,
|
|
streamText: jest.fn(),
|
|
generateObject: mockGenerateObject,
|
|
streamObject: jest.fn(),
|
|
zodSchema: jest.fn((schema) => schema),
|
|
NoObjectGeneratedError: mockNoObjectGeneratedError,
|
|
JSONParseError: mockJSONParseError
|
|
}));
|
|
|
|
// Mock jsonrepair
|
|
const mockJsonrepair = jest.fn();
|
|
jest.unstable_mockModule('jsonrepair', () => ({
|
|
jsonrepair: mockJsonrepair
|
|
}));
|
|
|
|
// Mock logging and utilities
|
|
jest.unstable_mockModule('../../../scripts/modules/utils.js', () => ({
|
|
log: jest.fn(),
|
|
findProjectRoot: jest.fn(() => '/mock/project/root'),
|
|
isEmpty: jest.fn(
|
|
(val) =>
|
|
!val ||
|
|
(Array.isArray(val) && val.length === 0) ||
|
|
(typeof val === 'object' && Object.keys(val).length === 0)
|
|
),
|
|
resolveEnvVariable: jest.fn((key) => process.env[key])
|
|
}));
|
|
|
|
// Import after mocking
|
|
const { BaseAIProvider } = await import(
|
|
'../../../src/ai-providers/base-provider.js'
|
|
);
|
|
|
|
describe('BaseAIProvider', () => {
|
|
let testProvider;
|
|
let mockClient;
|
|
|
|
beforeEach(() => {
|
|
// Create a concrete test provider
|
|
class TestProvider extends BaseAIProvider {
|
|
constructor() {
|
|
super();
|
|
this.name = 'TestProvider';
|
|
}
|
|
|
|
getRequiredApiKeyName() {
|
|
return 'TEST_API_KEY';
|
|
}
|
|
|
|
async getClient() {
|
|
return mockClient;
|
|
}
|
|
}
|
|
|
|
mockClient = jest.fn((modelId) => ({ modelId }));
|
|
jest.clearAllMocks();
|
|
testProvider = new TestProvider();
|
|
});
|
|
|
|
describe('1. Parameter Validation - Catches Invalid Inputs', () => {
|
|
describe('validateAuth', () => {
|
|
it('should throw when API key is missing', () => {
|
|
expect(() => testProvider.validateAuth({})).toThrow(
|
|
'TestProvider API key is required'
|
|
);
|
|
});
|
|
|
|
it('should pass when API key is provided', () => {
|
|
expect(() =>
|
|
testProvider.validateAuth({ apiKey: 'test-key' })
|
|
).not.toThrow();
|
|
});
|
|
});
|
|
|
|
describe('validateParams', () => {
|
|
it('should throw when model ID is missing', () => {
|
|
expect(() => testProvider.validateParams({ apiKey: 'key' })).toThrow(
|
|
'TestProvider Model ID is required'
|
|
);
|
|
});
|
|
|
|
it('should throw when both API key and model ID are missing', () => {
|
|
expect(() => testProvider.validateParams({})).toThrow(
|
|
'TestProvider API key is required'
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('validateOptionalParams', () => {
|
|
it('should throw for temperature below 0', () => {
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ temperature: -0.1 })
|
|
).toThrow('Temperature must be between 0 and 1');
|
|
});
|
|
|
|
it('should throw for temperature above 1', () => {
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ temperature: 1.1 })
|
|
).toThrow('Temperature must be between 0 and 1');
|
|
});
|
|
|
|
it('should accept temperature at boundaries', () => {
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ temperature: 0 })
|
|
).not.toThrow();
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ temperature: 1 })
|
|
).not.toThrow();
|
|
});
|
|
|
|
it('should throw for invalid maxTokens values', () => {
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ maxTokens: 0 })
|
|
).toThrow('maxTokens must be a finite number greater than 0');
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ maxTokens: -100 })
|
|
).toThrow('maxTokens must be a finite number greater than 0');
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ maxTokens: Infinity })
|
|
).toThrow('maxTokens must be a finite number greater than 0');
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ maxTokens: 'invalid' })
|
|
).toThrow('maxTokens must be a finite number greater than 0');
|
|
});
|
|
});
|
|
|
|
describe('validateMessages', () => {
|
|
it('should throw for null/undefined messages', async () => {
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: null
|
|
})
|
|
).rejects.toThrow('Invalid or empty messages array provided');
|
|
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: undefined
|
|
})
|
|
).rejects.toThrow('Invalid or empty messages array provided');
|
|
});
|
|
|
|
it('should throw for empty messages array', async () => {
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: []
|
|
})
|
|
).rejects.toThrow('Invalid or empty messages array provided');
|
|
});
|
|
|
|
it('should throw for messages without role or content', async () => {
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ content: 'test' }] // missing role
|
|
})
|
|
).rejects.toThrow(
|
|
'Invalid message format. Each message must have role and content'
|
|
);
|
|
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user' }] // missing content
|
|
})
|
|
).rejects.toThrow(
|
|
'Invalid message format. Each message must have role and content'
|
|
);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('2. Error Handling - Proper Error Context', () => {
|
|
it('should wrap API errors with context', async () => {
|
|
const apiError = new Error('API rate limit exceeded');
|
|
mockGenerateText.mockRejectedValue(apiError);
|
|
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
})
|
|
).rejects.toThrow(
|
|
'TestProvider API error during text generation: API rate limit exceeded'
|
|
);
|
|
});
|
|
|
|
it('should handle errors without message property', async () => {
|
|
const apiError = { code: 'NETWORK_ERROR' };
|
|
mockGenerateText.mockRejectedValue(apiError);
|
|
|
|
await expect(
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
})
|
|
).rejects.toThrow(
|
|
'TestProvider API error during text generation: Unknown error occurred'
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('3. Abstract Class Protection', () => {
|
|
it('should prevent direct instantiation of BaseAIProvider', () => {
|
|
expect(() => new BaseAIProvider()).toThrow(
|
|
'BaseAIProvider cannot be instantiated directly'
|
|
);
|
|
});
|
|
|
|
it('should throw when abstract methods are not implemented', () => {
|
|
class IncompleteProvider extends BaseAIProvider {
|
|
constructor() {
|
|
super();
|
|
}
|
|
}
|
|
const provider = new IncompleteProvider();
|
|
|
|
expect(() => provider.getClient()).toThrow(
|
|
'getClient must be implemented by provider'
|
|
);
|
|
expect(() => provider.getRequiredApiKeyName()).toThrow(
|
|
'getRequiredApiKeyName must be implemented by provider'
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('4. Token Parameter Preparation', () => {
|
|
it('should convert maxTokens to maxOutputTokens as integer', () => {
|
|
const result = testProvider.prepareTokenParam('model', 1000.7);
|
|
expect(result).toEqual({ maxOutputTokens: 1000 });
|
|
});
|
|
|
|
it('should handle string numbers', () => {
|
|
const result = testProvider.prepareTokenParam('model', '500');
|
|
expect(result).toEqual({ maxOutputTokens: 500 });
|
|
});
|
|
|
|
it('should return empty object when maxTokens is undefined', () => {
|
|
const result = testProvider.prepareTokenParam('model', undefined);
|
|
expect(result).toEqual({});
|
|
});
|
|
|
|
it('should floor decimal values', () => {
|
|
const result = testProvider.prepareTokenParam('model', 999.99);
|
|
expect(result).toEqual({ maxOutputTokens: 999 });
|
|
});
|
|
});
|
|
|
|
describe('5. JSON Repair for Malformed Responses', () => {
|
|
it('should repair malformed JSON in generateObject errors', async () => {
|
|
const malformedJson = '{"key": "value",,}'; // Double comma
|
|
const repairedJson = '{"key": "value"}';
|
|
|
|
const parseError = new mockJSONParseError(malformedJson);
|
|
const noObjectError = new mockNoObjectGeneratedError(parseError);
|
|
noObjectError.usage = {
|
|
promptTokens: 100,
|
|
completionTokens: 50,
|
|
totalTokens: 150
|
|
};
|
|
|
|
mockGenerateObject.mockRejectedValue(noObjectError);
|
|
mockJsonrepair.mockReturnValue(repairedJson);
|
|
|
|
const result = await testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
schema: { type: 'object' },
|
|
objectName: 'TestObject'
|
|
});
|
|
|
|
expect(mockJsonrepair).toHaveBeenCalledWith(malformedJson);
|
|
expect(result).toEqual({
|
|
object: { key: 'value' },
|
|
usage: {
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
totalTokens: 150
|
|
}
|
|
});
|
|
});
|
|
|
|
it('should throw original error when JSON repair fails', async () => {
|
|
const malformedJson = 'not even close to JSON';
|
|
const parseError = new mockJSONParseError(malformedJson);
|
|
const noObjectError = new mockNoObjectGeneratedError(parseError);
|
|
|
|
mockGenerateObject.mockRejectedValue(noObjectError);
|
|
mockJsonrepair.mockImplementation(() => {
|
|
throw new Error('Cannot repair this JSON');
|
|
});
|
|
|
|
await expect(
|
|
testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
schema: { type: 'object' },
|
|
objectName: 'TestObject'
|
|
})
|
|
).rejects.toThrow('TestProvider API error during object generation');
|
|
});
|
|
|
|
it('should handle non-JSON parse errors normally', async () => {
|
|
const regularError = new Error('Network timeout');
|
|
mockGenerateObject.mockRejectedValue(regularError);
|
|
|
|
await expect(
|
|
testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
schema: { type: 'object' },
|
|
objectName: 'TestObject'
|
|
})
|
|
).rejects.toThrow(
|
|
'TestProvider API error during object generation: Network timeout'
|
|
);
|
|
|
|
expect(mockJsonrepair).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('6. Usage Token Normalization', () => {
|
|
it('should normalize different token formats in generateText', async () => {
|
|
// Test promptTokens/completionTokens format (older format)
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { promptTokens: 10, completionTokens: 5 }
|
|
});
|
|
|
|
let result = await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
});
|
|
|
|
expect(result.usage).toEqual({
|
|
inputTokens: 10,
|
|
outputTokens: 5,
|
|
totalTokens: 15
|
|
});
|
|
|
|
// Test inputTokens/outputTokens format (newer format)
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 20, outputTokens: 10, totalTokens: 30 }
|
|
});
|
|
|
|
result = await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
});
|
|
|
|
expect(result.usage).toEqual({
|
|
inputTokens: 20,
|
|
outputTokens: 10,
|
|
totalTokens: 30
|
|
});
|
|
});
|
|
|
|
it('should handle missing usage data gracefully', async () => {
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: undefined
|
|
});
|
|
|
|
const result = await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
});
|
|
|
|
expect(result.usage).toEqual({
|
|
inputTokens: 0,
|
|
outputTokens: 0,
|
|
totalTokens: 0
|
|
});
|
|
});
|
|
|
|
it('should calculate totalTokens when missing', async () => {
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 15, outputTokens: 25 }
|
|
});
|
|
|
|
const result = await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
});
|
|
|
|
expect(result.usage.totalTokens).toBe(40);
|
|
});
|
|
});
|
|
|
|
describe('7. Schema Validation for Object Methods', () => {
|
|
it('should throw when schema is missing for generateObject', async () => {
|
|
await expect(
|
|
testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
objectName: 'TestObject'
|
|
// missing schema
|
|
})
|
|
).rejects.toThrow('Schema is required for object generation');
|
|
});
|
|
|
|
it('should throw when objectName is missing for generateObject', async () => {
|
|
await expect(
|
|
testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
schema: { type: 'object' }
|
|
// missing objectName
|
|
})
|
|
).rejects.toThrow('Object name is required for object generation');
|
|
});
|
|
|
|
it('should throw when schema is missing for streamObject', async () => {
|
|
await expect(
|
|
testProvider.streamObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
// missing schema
|
|
})
|
|
).rejects.toThrow('Schema is required for object streaming');
|
|
});
|
|
|
|
it('should use json mode when needsExplicitJsonSchema is true', async () => {
|
|
testProvider.needsExplicitJsonSchema = true;
|
|
mockGenerateObject.mockResolvedValue({
|
|
object: { test: 'value' },
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
await testProvider.generateObject({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
schema: { type: 'object' },
|
|
objectName: 'TestObject'
|
|
});
|
|
|
|
expect(mockGenerateObject).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
mode: 'json' // Should be 'json' not 'auto'
|
|
})
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('8. Integration Points - Client Creation', () => {
|
|
it('should pass params to getClient method', async () => {
|
|
const getClientSpy = jest.spyOn(testProvider, 'getClient');
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
const params = {
|
|
apiKey: 'test-key',
|
|
modelId: 'test-model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
customParam: 'custom-value'
|
|
};
|
|
|
|
await testProvider.generateText(params);
|
|
|
|
expect(getClientSpy).toHaveBeenCalledWith(params);
|
|
});
|
|
|
|
it('should use client with correct model ID', async () => {
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'gpt-4-turbo',
|
|
messages: [{ role: 'user', content: 'test' }]
|
|
});
|
|
|
|
expect(mockClient).toHaveBeenCalledWith('gpt-4-turbo');
|
|
expect(mockGenerateText).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
model: { modelId: 'gpt-4-turbo' }
|
|
})
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('9. Edge Cases - Boundary Conditions', () => {
|
|
it('should handle zero maxTokens gracefully', () => {
|
|
// This should throw in validation
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ maxTokens: 0 })
|
|
).toThrow('maxTokens must be a finite number greater than 0');
|
|
});
|
|
|
|
it('should handle very large maxTokens', () => {
|
|
const result = testProvider.prepareTokenParam('model', 999999999);
|
|
expect(result).toEqual({ maxOutputTokens: 999999999 });
|
|
});
|
|
|
|
it('should handle NaN temperature gracefully', () => {
|
|
// NaN fails the range check (NaN < 0 is false, NaN > 1 is also false)
|
|
// But NaN is not between 0 and 1, so we need to check the actual behavior
|
|
// The current implementation doesn't explicitly check for NaN,
|
|
// it passes because NaN < 0 and NaN > 1 are both false
|
|
expect(() =>
|
|
testProvider.validateOptionalParams({ temperature: NaN })
|
|
).not.toThrow();
|
|
// This is actually a bug - NaN should be rejected
|
|
// But we're testing current behavior, not desired behavior
|
|
});
|
|
|
|
it('should handle concurrent calls safely', async () => {
|
|
mockGenerateText.mockImplementation(async () => ({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
}));
|
|
|
|
const promises = Array.from({ length: 10 }, (_, i) =>
|
|
testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: `model-${i}`,
|
|
messages: [{ role: 'user', content: `test-${i}` }]
|
|
})
|
|
);
|
|
|
|
const results = await Promise.all(promises);
|
|
expect(results).toHaveLength(10);
|
|
expect(mockClient).toHaveBeenCalledTimes(10);
|
|
});
|
|
});
|
|
|
|
describe('10. Default Behavior - isRequiredApiKey', () => {
|
|
it('should return true by default for isRequiredApiKey', () => {
|
|
expect(testProvider.isRequiredApiKey()).toBe(true);
|
|
});
|
|
|
|
it('should allow override of isRequiredApiKey', () => {
|
|
class NoAuthProvider extends BaseAIProvider {
|
|
constructor() {
|
|
super();
|
|
}
|
|
isRequiredApiKey() {
|
|
return false;
|
|
}
|
|
validateAuth() {
|
|
// Override to not require API key
|
|
}
|
|
getClient() {
|
|
return mockClient;
|
|
}
|
|
getRequiredApiKeyName() {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const provider = new NoAuthProvider();
|
|
expect(provider.isRequiredApiKey()).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('11. Temperature Filtering - CLI vs Standard Providers', () => {
|
|
const mockStreamText = jest.fn();
|
|
const mockStreamObject = jest.fn();
|
|
|
|
beforeEach(() => {
|
|
mockStreamText.mockReset();
|
|
mockStreamObject.mockReset();
|
|
});
|
|
|
|
it('should include temperature in generateText when supported', async () => {
|
|
testProvider.supportsTemperature = true;
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
temperature: 0.7
|
|
});
|
|
|
|
expect(mockGenerateText).toHaveBeenCalledWith(
|
|
expect.objectContaining({ temperature: 0.7 })
|
|
);
|
|
});
|
|
|
|
it('should exclude temperature in generateText when not supported', async () => {
|
|
testProvider.supportsTemperature = false;
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
temperature: 0.7
|
|
});
|
|
|
|
const callArgs = mockGenerateText.mock.calls[0][0];
|
|
expect(callArgs).not.toHaveProperty('temperature');
|
|
});
|
|
|
|
it('should exclude temperature when undefined even if supported', async () => {
|
|
testProvider.supportsTemperature = true;
|
|
mockGenerateText.mockResolvedValue({
|
|
text: 'response',
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
|
|
});
|
|
|
|
await testProvider.generateText({
|
|
apiKey: 'key',
|
|
modelId: 'model',
|
|
messages: [{ role: 'user', content: 'test' }],
|
|
temperature: undefined
|
|
});
|
|
|
|
const callArgs = mockGenerateText.mock.calls[0][0];
|
|
expect(callArgs).not.toHaveProperty('temperature');
|
|
});
|
|
});
|
|
});
|