120 lines
4.9 KiB
Plaintext
120 lines
4.9 KiB
Plaintext
# Task ID: 87
|
|
# Title: Implement validation and error handling
|
|
# Status: pending
|
|
# Dependencies: 85
|
|
# Priority: low
|
|
# Description: Add comprehensive validation and error handling for token limits throughout the system, including helpful error messages and graceful fallbacks.
|
|
# Details:
|
|
1. Add validation when loading models in `config-manager.js`:
|
|
```javascript
|
|
function _validateModelMap(modelMap) {
|
|
// Validate each provider's models
|
|
Object.entries(modelMap).forEach(([provider, models]) => {
|
|
models.forEach(model => {
|
|
// Check for required token limit fields
|
|
if (!model.contextWindowTokens) {
|
|
console.warn(`Warning: Model ${model.id} from ${provider} is missing contextWindowTokens field`);
|
|
}
|
|
if (!model.maxOutputTokens) {
|
|
console.warn(`Warning: Model ${model.id} from ${provider} is missing maxOutputTokens field`);
|
|
}
|
|
});
|
|
});
|
|
return modelMap;
|
|
}
|
|
```
|
|
|
|
2. Add validation when setting up a model in the CLI:
|
|
```javascript
|
|
function validateModelConfig(modelConfig, modelCapabilities) {
|
|
const issues = [];
|
|
|
|
// Check if input tokens exceed model's context window
|
|
if (modelConfig.maxInputTokens > modelCapabilities.contextWindowTokens) {
|
|
issues.push(`maxInputTokens (${modelConfig.maxInputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);
|
|
}
|
|
|
|
// Check if output tokens exceed model's maximum
|
|
if (modelConfig.maxOutputTokens > modelCapabilities.maxOutputTokens) {
|
|
issues.push(`maxOutputTokens (${modelConfig.maxOutputTokens}) exceeds model's maximum output tokens (${modelCapabilities.maxOutputTokens})`);
|
|
}
|
|
|
|
// Check if combined tokens exceed context window
|
|
if (modelConfig.maxInputTokens + modelConfig.maxOutputTokens > modelCapabilities.contextWindowTokens) {
|
|
issues.push(`Combined maxInputTokens and maxOutputTokens (${modelConfig.maxInputTokens + modelConfig.maxOutputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);
|
|
}
|
|
|
|
return issues;
|
|
}
|
|
```
|
|
|
|
3. Add graceful fallbacks in `ai-services-unified.js`:
|
|
```javascript
|
|
// Fallback for missing token limits
|
|
if (!roleParams.maxInputTokens) {
|
|
console.warn(`Warning: maxInputTokens not specified for role '${currentRole}'. Using default value.`);
|
|
roleParams.maxInputTokens = 8000; // Reasonable default
|
|
}
|
|
|
|
if (!roleParams.maxOutputTokens) {
|
|
console.warn(`Warning: maxOutputTokens not specified for role '${currentRole}'. Using default value.`);
|
|
roleParams.maxOutputTokens = 2000; // Reasonable default
|
|
}
|
|
|
|
// Fallback for missing model capabilities
|
|
if (!modelCapabilities.contextWindowTokens) {
|
|
console.warn(`Warning: contextWindowTokens not specified for model ${modelId}. Using conservative estimate.`);
|
|
modelCapabilities.contextWindowTokens = roleParams.maxInputTokens + roleParams.maxOutputTokens;
|
|
}
|
|
|
|
if (!modelCapabilities.maxOutputTokens) {
|
|
console.warn(`Warning: maxOutputTokens not specified for model ${modelId}. Using role configuration.`);
|
|
modelCapabilities.maxOutputTokens = roleParams.maxOutputTokens;
|
|
}
|
|
```
|
|
|
|
4. Add detailed logging for token usage:
|
|
```javascript
|
|
function logTokenUsage(provider, modelId, inputTokens, outputTokens, role) {
|
|
const inputCost = calculateTokenCost(provider, modelId, 'input', inputTokens);
|
|
const outputCost = calculateTokenCost(provider, modelId, 'output', outputTokens);
|
|
|
|
console.info(`Token usage for ${role} role with ${provider}/${modelId}:`);
|
|
console.info(`- Input: ${inputTokens.toLocaleString()} tokens ($${inputCost.toFixed(6)})`);
|
|
console.info(`- Output: ${outputTokens.toLocaleString()} tokens ($${outputCost.toFixed(6)})`);
|
|
console.info(`- Total cost: $${(inputCost + outputCost).toFixed(6)}`);
|
|
console.info(`- Available output tokens: ${availableOutputTokens.toLocaleString()}`);
|
|
}
|
|
```
|
|
|
|
5. Add a helper function to suggest configuration improvements:
|
|
```javascript
|
|
function suggestTokenConfigImprovements(roleParams, modelCapabilities, promptTokens) {
|
|
const suggestions = [];
|
|
|
|
// If prompt is using less than 50% of allowed input
|
|
if (promptTokens < roleParams.maxInputTokens * 0.5) {
|
|
suggestions.push(`Consider reducing maxInputTokens from ${roleParams.maxInputTokens} to save on potential costs`);
|
|
}
|
|
|
|
// If output tokens are very limited due to large input
|
|
const availableOutput = Math.min(
|
|
roleParams.maxOutputTokens,
|
|
modelCapabilities.contextWindowTokens - promptTokens
|
|
);
|
|
|
|
if (availableOutput < roleParams.maxOutputTokens * 0.5) {
|
|
suggestions.push(`Available output tokens (${availableOutput}) are significantly less than configured maxOutputTokens (${roleParams.maxOutputTokens}) due to large input`);
|
|
}
|
|
|
|
return suggestions;
|
|
}
|
|
```
|
|
|
|
# Test Strategy:
|
|
1. Test validation functions with valid and invalid configurations
|
|
2. Verify fallback behavior works correctly when configuration is missing
|
|
3. Test error messages are clear and actionable
|
|
4. Test logging functions provide useful information
|
|
5. Verify suggestion logic provides helpful recommendations
|