fix(ai): Correctly imports generateText in openai.js, adds specific cause and reason for OpenRouter failures in the openrouter.js catch, performs complexity analysis on all tm tasks, adds new tasks to further improve the maxTokens to take input and output maximum into account. Adjusts default fallback max tokens so 3.5 does not fail.
This commit is contained in:
Eyal Toledano
34
tasks/task_082.txt
Normal file
34
tasks/task_082.txt
Normal file
@@ -0,0 +1,34 @@
|
||||
# Task ID: 82
|
||||
# Title: Update supported-models.json with token limit fields
|
||||
# Status: pending
|
||||
# Dependencies: None
|
||||
# Priority: high
|
||||
# Description: Modify the supported-models.json file to include contextWindowTokens and maxOutputTokens fields for each model, replacing the ambiguous max_tokens field.
|
||||
# Details:
|
||||
For each model entry in supported-models.json:
|
||||
1. Add `contextWindowTokens` field representing the total context window (input + output tokens)
|
||||
2. Add `maxOutputTokens` field representing the maximum tokens the model can generate
|
||||
3. Remove or deprecate the ambiguous `max_tokens` field if present
|
||||
|
||||
Research and populate accurate values for each model from official documentation:
|
||||
- For OpenAI models (e.g., gpt-4o): contextWindowTokens=128000, maxOutputTokens=16384
|
||||
- For Anthropic models (e.g., Claude 3.7): contextWindowTokens=200000, maxOutputTokens=8192
|
||||
- For other providers, find official documentation or use reasonable defaults
|
||||
|
||||
Example entry:
|
||||
```json
|
||||
{
|
||||
"id": "claude-3-7-sonnet-20250219",
|
||||
"swe_score": 0.623,
|
||||
"cost_per_1m_tokens": { "input": 3.0, "output": 15.0 },
|
||||
"allowed_roles": ["main", "fallback"],
|
||||
"contextWindowTokens": 200000,
|
||||
"maxOutputTokens": 8192
|
||||
}
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Validate JSON syntax after changes
|
||||
2. Verify all models have the new fields with reasonable values
|
||||
3. Check that the values align with official documentation from each provider
|
||||
4. Ensure backward compatibility by maintaining any fields other systems might depend on
|
||||
95
tasks/task_083.txt
Normal file
95
tasks/task_083.txt
Normal file
@@ -0,0 +1,95 @@
|
||||
# Task ID: 83
|
||||
# Title: Update config-manager.js defaults and getters
|
||||
# Status: pending
|
||||
# Dependencies: 82
|
||||
# Priority: high
|
||||
# Description: Modify the config-manager.js module to replace maxTokens with maxInputTokens and maxOutputTokens in the DEFAULTS object and update related getter functions.
|
||||
# Details:
|
||||
1. Update the `DEFAULTS` object in config-manager.js:
|
||||
```javascript
|
||||
const DEFAULTS = {
|
||||
// ... existing defaults
|
||||
main: {
|
||||
// Replace maxTokens with these two fields
|
||||
maxInputTokens: 16000, // Example default
|
||||
maxOutputTokens: 4000, // Example default
|
||||
temperature: 0.7
|
||||
// ... other fields
|
||||
},
|
||||
research: {
|
||||
maxInputTokens: 16000,
|
||||
maxOutputTokens: 4000,
|
||||
temperature: 0.7
|
||||
// ... other fields
|
||||
},
|
||||
fallback: {
|
||||
maxInputTokens: 8000,
|
||||
maxOutputTokens: 2000,
|
||||
temperature: 0.7
|
||||
// ... other fields
|
||||
}
|
||||
// ... rest of DEFAULTS
|
||||
};
|
||||
```
|
||||
|
||||
2. Update `getParametersForRole` function to return the new fields:
|
||||
```javascript
|
||||
function getParametersForRole(role, explicitRoot = null) {
|
||||
const config = _getConfig(explicitRoot);
|
||||
return {
|
||||
maxInputTokens: config[role]?.maxInputTokens,
|
||||
maxOutputTokens: config[role]?.maxOutputTokens,
|
||||
temperature: config[role]?.temperature
|
||||
// ... any other parameters
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
3. Add a new function to get model capabilities:
|
||||
```javascript
|
||||
function getModelCapabilities(providerName, modelId) {
|
||||
const models = MODEL_MAP[providerName?.toLowerCase()];
|
||||
const model = models?.find(m => m.id === modelId);
|
||||
return {
|
||||
contextWindowTokens: model?.contextWindowTokens,
|
||||
maxOutputTokens: model?.maxOutputTokens
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
4. Deprecate or update the role-specific maxTokens getters:
|
||||
```javascript
|
||||
// Either remove these or update them to return maxInputTokens
|
||||
function getMainMaxTokens(explicitRoot = null) {
|
||||
console.warn('getMainMaxTokens is deprecated. Use getParametersForRole("main") instead.');
|
||||
return getParametersForRole("main", explicitRoot).maxInputTokens;
|
||||
}
|
||||
// Same for getResearchMaxTokens and getFallbackMaxTokens
|
||||
```
|
||||
|
||||
5. Export the new functions:
|
||||
```javascript
|
||||
module.exports = {
|
||||
// ... existing exports
|
||||
getParametersForRole,
|
||||
getModelCapabilities
|
||||
};
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Unit test the updated getParametersForRole function with various configurations
|
||||
2. Verify the new getModelCapabilities function returns correct values
|
||||
3. Test with both default and custom configurations
|
||||
4. Ensure backward compatibility by checking that existing code using the old getters still works (with warnings)
|
||||
|
||||
# Subtasks:
|
||||
## 1. Update config-manager.js with specific token limit fields [pending]
|
||||
### Dependencies: None
|
||||
### Description: Modify the DEFAULTS object in config-manager.js to replace maxTokens with more specific token limit fields (maxInputTokens, maxOutputTokens, maxTotalTokens) and update related getter functions while maintaining backward compatibility.
|
||||
### Details:
|
||||
1. Replace maxTokens in the DEFAULTS object with maxInputTokens, maxOutputTokens, and maxTotalTokens
|
||||
2. Update any getter functions that reference maxTokens to handle both old and new configurations
|
||||
3. Ensure backward compatibility so existing code using maxTokens continues to work
|
||||
4. Update any related documentation or comments to reflect the new token limit fields
|
||||
5. Test the changes to verify both new specific token limits and legacy maxTokens usage work correctly
|
||||
|
||||
93
tasks/task_084.txt
Normal file
93
tasks/task_084.txt
Normal file
@@ -0,0 +1,93 @@
|
||||
# Task ID: 84
|
||||
# Title: Implement token counting utility
|
||||
# Status: pending
|
||||
# Dependencies: 82
|
||||
# Priority: high
|
||||
# Description: Create a utility function to count tokens for prompts based on the model being used, primarily using tiktoken for OpenAI and Anthropic models with character-based fallbacks for other providers.
|
||||
# Details:
|
||||
1. Install the tiktoken package:
|
||||
```bash
|
||||
npm install tiktoken
|
||||
```
|
||||
|
||||
2. Create a new file `scripts/modules/token-counter.js`:
|
||||
```javascript
|
||||
const tiktoken = require('tiktoken');
|
||||
|
||||
/**
|
||||
* Count tokens for a given text and model
|
||||
* @param {string} text - The text to count tokens for
|
||||
* @param {string} provider - The AI provider (e.g., 'openai', 'anthropic')
|
||||
* @param {string} modelId - The model ID
|
||||
* @returns {number} - Estimated token count
|
||||
*/
|
||||
function countTokens(text, provider, modelId) {
|
||||
if (!text) return 0;
|
||||
|
||||
// Convert to lowercase for case-insensitive matching
|
||||
const providerLower = provider?.toLowerCase();
|
||||
|
||||
try {
|
||||
// OpenAI models
|
||||
if (providerLower === 'openai') {
|
||||
// Most OpenAI chat models use cl100k_base encoding
|
||||
const encoding = tiktoken.encoding_for_model(modelId) || tiktoken.get_encoding('cl100k_base');
|
||||
return encoding.encode(text).length;
|
||||
}
|
||||
|
||||
// Anthropic models - can use cl100k_base as an approximation
|
||||
// or follow Anthropic's guidance
|
||||
if (providerLower === 'anthropic') {
|
||||
try {
|
||||
// Try to use cl100k_base as a reasonable approximation
|
||||
const encoding = tiktoken.get_encoding('cl100k_base');
|
||||
return encoding.encode(text).length;
|
||||
} catch (e) {
|
||||
// Fallback to Anthropic's character-based estimation
|
||||
return Math.ceil(text.length / 3.5); // ~3.5 chars per token for English
|
||||
}
|
||||
}
|
||||
|
||||
// For other providers, use character-based estimation as fallback
|
||||
// Different providers may have different tokenization schemes
|
||||
return Math.ceil(text.length / 4); // General fallback estimate
|
||||
} catch (error) {
|
||||
console.warn(`Token counting error: ${error.message}. Using character-based estimate.`);
|
||||
return Math.ceil(text.length / 4); // Fallback if tiktoken fails
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { countTokens };
|
||||
```
|
||||
|
||||
3. Add tests for the token counter in `tests/token-counter.test.js`:
|
||||
```javascript
|
||||
const { countTokens } = require('../scripts/modules/token-counter');
|
||||
|
||||
describe('Token Counter', () => {
|
||||
test('counts tokens for OpenAI models', () => {
|
||||
const text = 'Hello, world! This is a test.';
|
||||
const count = countTokens(text, 'openai', 'gpt-4');
|
||||
expect(count).toBeGreaterThan(0);
|
||||
expect(typeof count).toBe('number');
|
||||
});
|
||||
|
||||
test('counts tokens for Anthropic models', () => {
|
||||
const text = 'Hello, world! This is a test.';
|
||||
const count = countTokens(text, 'anthropic', 'claude-3-7-sonnet-20250219');
|
||||
expect(count).toBeGreaterThan(0);
|
||||
expect(typeof count).toBe('number');
|
||||
});
|
||||
|
||||
test('handles empty text', () => {
|
||||
expect(countTokens('', 'openai', 'gpt-4')).toBe(0);
|
||||
expect(countTokens(null, 'openai', 'gpt-4')).toBe(0);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Unit test the countTokens function with various inputs and models
|
||||
2. Compare token counts with known examples from OpenAI and Anthropic documentation
|
||||
3. Test edge cases: empty strings, very long texts, non-English texts
|
||||
4. Test fallback behavior when tiktoken fails or is not applicable
|
||||
104
tasks/task_085.txt
Normal file
104
tasks/task_085.txt
Normal file
@@ -0,0 +1,104 @@
|
||||
# Task ID: 85
|
||||
# Title: Update ai-services-unified.js for dynamic token limits
|
||||
# Status: pending
|
||||
# Dependencies: 83, 84
|
||||
# Priority: medium
|
||||
# Description: Modify the _unifiedServiceRunner function in ai-services-unified.js to use the new token counting utility and dynamically adjust output token limits based on input length.
|
||||
# Details:
|
||||
1. Import the token counter in `ai-services-unified.js`:
|
||||
```javascript
|
||||
const { countTokens } = require('./token-counter');
|
||||
const { getParametersForRole, getModelCapabilities } = require('./config-manager');
|
||||
```
|
||||
|
||||
2. Update the `_unifiedServiceRunner` function to implement dynamic token limit adjustment:
|
||||
```javascript
|
||||
async function _unifiedServiceRunner({
|
||||
serviceType,
|
||||
provider,
|
||||
modelId,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
currentRole,
|
||||
effectiveProjectRoot,
|
||||
// ... other parameters
|
||||
}) {
|
||||
// Get role parameters with new token limits
|
||||
const roleParams = getParametersForRole(currentRole, effectiveProjectRoot);
|
||||
|
||||
// Get model capabilities
|
||||
const modelCapabilities = getModelCapabilities(provider, modelId);
|
||||
|
||||
// Count tokens in the prompts
|
||||
const systemPromptTokens = countTokens(systemPrompt, provider, modelId);
|
||||
const userPromptTokens = countTokens(prompt, provider, modelId);
|
||||
const totalPromptTokens = systemPromptTokens + userPromptTokens;
|
||||
|
||||
// Validate against input token limits
|
||||
if (totalPromptTokens > roleParams.maxInputTokens) {
|
||||
throw new Error(
|
||||
`Prompt (${totalPromptTokens} tokens) exceeds configured max input tokens (${roleParams.maxInputTokens}) for role '${currentRole}'.`
|
||||
);
|
||||
}
|
||||
|
||||
// Validate against model's absolute context window
|
||||
if (modelCapabilities.contextWindowTokens && totalPromptTokens > modelCapabilities.contextWindowTokens) {
|
||||
throw new Error(
|
||||
`Prompt (${totalPromptTokens} tokens) exceeds model's context window (${modelCapabilities.contextWindowTokens}) for ${modelId}.`
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate available output tokens
|
||||
// If model has a combined context window, we need to subtract input tokens
|
||||
let availableOutputTokens = roleParams.maxOutputTokens;
|
||||
|
||||
// If model has a context window constraint, ensure we don't exceed it
|
||||
if (modelCapabilities.contextWindowTokens) {
|
||||
const remainingContextTokens = modelCapabilities.contextWindowTokens - totalPromptTokens;
|
||||
availableOutputTokens = Math.min(availableOutputTokens, remainingContextTokens);
|
||||
}
|
||||
|
||||
// Also respect the model's absolute max output limit
|
||||
if (modelCapabilities.maxOutputTokens) {
|
||||
availableOutputTokens = Math.min(availableOutputTokens, modelCapabilities.maxOutputTokens);
|
||||
}
|
||||
|
||||
// Prepare API call parameters
|
||||
const callParams = {
|
||||
apiKey,
|
||||
modelId,
|
||||
maxTokens: availableOutputTokens, // Use dynamically calculated output limit
|
||||
temperature: roleParams.temperature,
|
||||
messages,
|
||||
baseUrl,
|
||||
...(serviceType === 'generateObject' && { schema, objectName }),
|
||||
...restApiParams
|
||||
};
|
||||
|
||||
// Log token usage information
|
||||
console.debug(`Token usage: ${totalPromptTokens} input tokens, ${availableOutputTokens} max output tokens`);
|
||||
|
||||
// Rest of the function remains the same...
|
||||
}
|
||||
```
|
||||
|
||||
3. Update the error handling to provide clear messages about token limits:
|
||||
```javascript
|
||||
try {
|
||||
// Existing code...
|
||||
} catch (error) {
|
||||
if (error.message.includes('tokens')) {
|
||||
// Token-related errors should be clearly identified
|
||||
console.error(`Token limit error: ${error.message}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Test with prompts of various lengths to verify dynamic adjustment
|
||||
2. Test with different models to ensure model-specific limits are respected
|
||||
3. Verify error messages are clear when limits are exceeded
|
||||
4. Test edge cases: very short prompts, prompts near the limit
|
||||
5. Integration test with actual API calls to verify the calculated limits work in practice
|
||||
107
tasks/task_086.txt
Normal file
107
tasks/task_086.txt
Normal file
@@ -0,0 +1,107 @@
|
||||
# Task ID: 86
|
||||
# Title: Update .taskmasterconfig schema and user guide
|
||||
# Status: pending
|
||||
# Dependencies: 83
|
||||
# Priority: medium
|
||||
# Description: Create a migration guide for users to update their .taskmasterconfig files and document the new token limit configuration options.
|
||||
# Details:
|
||||
1. Create a migration script or guide for users to update their existing `.taskmasterconfig` files:
|
||||
|
||||
```javascript
|
||||
// Example migration snippet for .taskmasterconfig
|
||||
{
|
||||
"main": {
|
||||
// Before:
|
||||
// "maxTokens": 16000,
|
||||
|
||||
// After:
|
||||
"maxInputTokens": 16000,
|
||||
"maxOutputTokens": 4000,
|
||||
"temperature": 0.7
|
||||
},
|
||||
"research": {
|
||||
"maxInputTokens": 16000,
|
||||
"maxOutputTokens": 4000,
|
||||
"temperature": 0.7
|
||||
},
|
||||
"fallback": {
|
||||
"maxInputTokens": 8000,
|
||||
"maxOutputTokens": 2000,
|
||||
"temperature": 0.7
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. Update the user documentation to explain the new token limit fields:
|
||||
|
||||
```markdown
|
||||
# Token Limit Configuration
|
||||
|
||||
Task Master now provides more granular control over token limits with separate settings for input and output tokens:
|
||||
|
||||
- `maxInputTokens`: Maximum number of tokens allowed in the input prompt (system prompt + user prompt)
|
||||
- `maxOutputTokens`: Maximum number of tokens the model should generate in its response
|
||||
|
||||
## Benefits
|
||||
|
||||
- More precise control over token usage
|
||||
- Better cost management
|
||||
- Reduced likelihood of hitting model context limits
|
||||
- Dynamic adjustment to maximize output space based on input length
|
||||
|
||||
## Migration from Previous Versions
|
||||
|
||||
If you're upgrading from a previous version, you'll need to update your `.taskmasterconfig` file:
|
||||
|
||||
1. Replace the single `maxTokens` field with separate `maxInputTokens` and `maxOutputTokens` fields
|
||||
2. Recommended starting values:
|
||||
- Set `maxInputTokens` to your previous `maxTokens` value
|
||||
- Set `maxOutputTokens` to approximately 1/4 of your model's context window
|
||||
|
||||
## Example Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"main": {
|
||||
"maxInputTokens": 16000,
|
||||
"maxOutputTokens": 4000,
|
||||
"temperature": 0.7
|
||||
}
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
3. Update the schema validation in `config-manager.js` to validate the new fields:
|
||||
|
||||
```javascript
|
||||
function _validateConfig(config) {
|
||||
// ... existing validation
|
||||
|
||||
// Validate token limits for each role
|
||||
['main', 'research', 'fallback'].forEach(role => {
|
||||
if (config[role]) {
|
||||
// Check if old maxTokens is present and warn about migration
|
||||
if (config[role].maxTokens !== undefined) {
|
||||
console.warn(`Warning: 'maxTokens' in ${role} role is deprecated. Please use 'maxInputTokens' and 'maxOutputTokens' instead.`);
|
||||
}
|
||||
|
||||
// Validate new token limit fields
|
||||
if (config[role].maxInputTokens !== undefined && (!Number.isInteger(config[role].maxInputTokens) || config[role].maxInputTokens <= 0)) {
|
||||
throw new Error(`Invalid maxInputTokens for ${role} role: must be a positive integer`);
|
||||
}
|
||||
|
||||
if (config[role].maxOutputTokens !== undefined && (!Number.isInteger(config[role].maxOutputTokens) || config[role].maxOutputTokens <= 0)) {
|
||||
throw new Error(`Invalid maxOutputTokens for ${role} role: must be a positive integer`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
}
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Verify documentation is clear and provides migration steps
|
||||
2. Test the validation logic with various config formats
|
||||
3. Test backward compatibility with old config format
|
||||
4. Ensure error messages are helpful when validation fails
|
||||
119
tasks/task_087.txt
Normal file
119
tasks/task_087.txt
Normal file
@@ -0,0 +1,119 @@
|
||||
# Task ID: 87
|
||||
# Title: Implement validation and error handling
|
||||
# Status: pending
|
||||
# Dependencies: 85
|
||||
# Priority: low
|
||||
# Description: Add comprehensive validation and error handling for token limits throughout the system, including helpful error messages and graceful fallbacks.
|
||||
# Details:
|
||||
1. Add validation when loading models in `config-manager.js`:
|
||||
```javascript
|
||||
function _validateModelMap(modelMap) {
|
||||
// Validate each provider's models
|
||||
Object.entries(modelMap).forEach(([provider, models]) => {
|
||||
models.forEach(model => {
|
||||
// Check for required token limit fields
|
||||
if (!model.contextWindowTokens) {
|
||||
console.warn(`Warning: Model ${model.id} from ${provider} is missing contextWindowTokens field`);
|
||||
}
|
||||
if (!model.maxOutputTokens) {
|
||||
console.warn(`Warning: Model ${model.id} from ${provider} is missing maxOutputTokens field`);
|
||||
}
|
||||
});
|
||||
});
|
||||
return modelMap;
|
||||
}
|
||||
```
|
||||
|
||||
2. Add validation when setting up a model in the CLI:
|
||||
```javascript
|
||||
function validateModelConfig(modelConfig, modelCapabilities) {
|
||||
const issues = [];
|
||||
|
||||
// Check if input tokens exceed model's context window
|
||||
if (modelConfig.maxInputTokens > modelCapabilities.contextWindowTokens) {
|
||||
issues.push(`maxInputTokens (${modelConfig.maxInputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);
|
||||
}
|
||||
|
||||
// Check if output tokens exceed model's maximum
|
||||
if (modelConfig.maxOutputTokens > modelCapabilities.maxOutputTokens) {
|
||||
issues.push(`maxOutputTokens (${modelConfig.maxOutputTokens}) exceeds model's maximum output tokens (${modelCapabilities.maxOutputTokens})`);
|
||||
}
|
||||
|
||||
// Check if combined tokens exceed context window
|
||||
if (modelConfig.maxInputTokens + modelConfig.maxOutputTokens > modelCapabilities.contextWindowTokens) {
|
||||
issues.push(`Combined maxInputTokens and maxOutputTokens (${modelConfig.maxInputTokens + modelConfig.maxOutputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
```
|
||||
|
||||
3. Add graceful fallbacks in `ai-services-unified.js`:
|
||||
```javascript
|
||||
// Fallback for missing token limits
|
||||
if (!roleParams.maxInputTokens) {
|
||||
console.warn(`Warning: maxInputTokens not specified for role '${currentRole}'. Using default value.`);
|
||||
roleParams.maxInputTokens = 8000; // Reasonable default
|
||||
}
|
||||
|
||||
if (!roleParams.maxOutputTokens) {
|
||||
console.warn(`Warning: maxOutputTokens not specified for role '${currentRole}'. Using default value.`);
|
||||
roleParams.maxOutputTokens = 2000; // Reasonable default
|
||||
}
|
||||
|
||||
// Fallback for missing model capabilities
|
||||
if (!modelCapabilities.contextWindowTokens) {
|
||||
console.warn(`Warning: contextWindowTokens not specified for model ${modelId}. Using conservative estimate.`);
|
||||
modelCapabilities.contextWindowTokens = roleParams.maxInputTokens + roleParams.maxOutputTokens;
|
||||
}
|
||||
|
||||
if (!modelCapabilities.maxOutputTokens) {
|
||||
console.warn(`Warning: maxOutputTokens not specified for model ${modelId}. Using role configuration.`);
|
||||
modelCapabilities.maxOutputTokens = roleParams.maxOutputTokens;
|
||||
}
|
||||
```
|
||||
|
||||
4. Add detailed logging for token usage:
|
||||
```javascript
|
||||
function logTokenUsage(provider, modelId, inputTokens, outputTokens, role) {
|
||||
const inputCost = calculateTokenCost(provider, modelId, 'input', inputTokens);
|
||||
const outputCost = calculateTokenCost(provider, modelId, 'output', outputTokens);
|
||||
|
||||
console.info(`Token usage for ${role} role with ${provider}/${modelId}:`);
|
||||
console.info(`- Input: ${inputTokens.toLocaleString()} tokens ($${inputCost.toFixed(6)})`);
|
||||
console.info(`- Output: ${outputTokens.toLocaleString()} tokens ($${outputCost.toFixed(6)})`);
|
||||
console.info(`- Total cost: $${(inputCost + outputCost).toFixed(6)}`);
|
||||
console.info(`- Available output tokens: ${availableOutputTokens.toLocaleString()}`);
|
||||
}
|
||||
```
|
||||
|
||||
5. Add a helper function to suggest configuration improvements:
|
||||
```javascript
|
||||
function suggestTokenConfigImprovements(roleParams, modelCapabilities, promptTokens) {
|
||||
const suggestions = [];
|
||||
|
||||
// If prompt is using less than 50% of allowed input
|
||||
if (promptTokens < roleParams.maxInputTokens * 0.5) {
|
||||
suggestions.push(`Consider reducing maxInputTokens from ${roleParams.maxInputTokens} to save on potential costs`);
|
||||
}
|
||||
|
||||
// If output tokens are very limited due to large input
|
||||
const availableOutput = Math.min(
|
||||
roleParams.maxOutputTokens,
|
||||
modelCapabilities.contextWindowTokens - promptTokens
|
||||
);
|
||||
|
||||
if (availableOutput < roleParams.maxOutputTokens * 0.5) {
|
||||
suggestions.push(`Available output tokens (${availableOutput}) are significantly less than configured maxOutputTokens (${roleParams.maxOutputTokens}) due to large input`);
|
||||
}
|
||||
|
||||
return suggestions;
|
||||
}
|
||||
```
|
||||
|
||||
# Test Strategy:
|
||||
1. Test validation functions with valid and invalid configurations
|
||||
2. Verify fallback behavior works correctly when configuration is missing
|
||||
3. Test error messages are clear and actionable
|
||||
4. Test logging functions provide useful information
|
||||
5. Verify suggestion logic provides helpful recommendations
|
||||
@@ -1,12 +1,4 @@
|
||||
{
|
||||
"meta": {
|
||||
"projectName": "Your Project Name",
|
||||
"version": "1.0.0",
|
||||
"source": "scripts/prd.txt",
|
||||
"description": "Tasks generated from PRD",
|
||||
"totalTasksGenerated": 20,
|
||||
"tasksIncluded": 20
|
||||
},
|
||||
"tasks": [
|
||||
{
|
||||
"id": 1,
|
||||
@@ -5228,6 +5220,92 @@
|
||||
"testStrategy": "Test dashboard visualizations with various usage patterns. Verify recommendations are relevant and helpful based on simulated usage data. Conduct usability testing to ensure insights are presented in an understandable way. Test with different user profiles to ensure recommendations are appropriately personalized."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 82,
|
||||
"title": "Update supported-models.json with token limit fields",
|
||||
"description": "Modify the supported-models.json file to include contextWindowTokens and maxOutputTokens fields for each model, replacing the ambiguous max_tokens field.",
|
||||
"details": "For each model entry in supported-models.json:\n1. Add `contextWindowTokens` field representing the total context window (input + output tokens)\n2. Add `maxOutputTokens` field representing the maximum tokens the model can generate\n3. Remove or deprecate the ambiguous `max_tokens` field if present\n\nResearch and populate accurate values for each model from official documentation:\n- For OpenAI models (e.g., gpt-4o): contextWindowTokens=128000, maxOutputTokens=16384\n- For Anthropic models (e.g., Claude 3.7): contextWindowTokens=200000, maxOutputTokens=8192\n- For other providers, find official documentation or use reasonable defaults\n\nExample entry:\n```json\n{\n \"id\": \"claude-3-7-sonnet-20250219\",\n \"swe_score\": 0.623,\n \"cost_per_1m_tokens\": { \"input\": 3.0, \"output\": 15.0 },\n \"allowed_roles\": [\"main\", \"fallback\"],\n \"contextWindowTokens\": 200000,\n \"maxOutputTokens\": 8192\n}\n```",
|
||||
"testStrategy": "1. Validate JSON syntax after changes\n2. Verify all models have the new fields with reasonable values\n3. Check that the values align with official documentation from each provider\n4. Ensure backward compatibility by maintaining any fields other systems might depend on",
|
||||
"priority": "high",
|
||||
"dependencies": [],
|
||||
"status": "pending",
|
||||
"subtasks": []
|
||||
},
|
||||
{
|
||||
"id": 83,
|
||||
"title": "Update config-manager.js defaults and getters",
|
||||
"description": "Modify the config-manager.js module to replace maxTokens with maxInputTokens and maxOutputTokens in the DEFAULTS object and update related getter functions.",
|
||||
"details": "1. Update the `DEFAULTS` object in config-manager.js:\n```javascript\nconst DEFAULTS = {\n // ... existing defaults\n main: {\n // Replace maxTokens with these two fields\n maxInputTokens: 16000, // Example default\n maxOutputTokens: 4000, // Example default\n temperature: 0.7\n // ... other fields\n },\n research: {\n maxInputTokens: 16000,\n maxOutputTokens: 4000,\n temperature: 0.7\n // ... other fields\n },\n fallback: {\n maxInputTokens: 8000,\n maxOutputTokens: 2000,\n temperature: 0.7\n // ... other fields\n }\n // ... rest of DEFAULTS\n};\n```\n\n2. Update `getParametersForRole` function to return the new fields:\n```javascript\nfunction getParametersForRole(role, explicitRoot = null) {\n const config = _getConfig(explicitRoot);\n return {\n maxInputTokens: config[role]?.maxInputTokens,\n maxOutputTokens: config[role]?.maxOutputTokens,\n temperature: config[role]?.temperature\n // ... any other parameters\n };\n}\n```\n\n3. Add a new function to get model capabilities:\n```javascript\nfunction getModelCapabilities(providerName, modelId) {\n const models = MODEL_MAP[providerName?.toLowerCase()];\n const model = models?.find(m => m.id === modelId);\n return {\n contextWindowTokens: model?.contextWindowTokens,\n maxOutputTokens: model?.maxOutputTokens\n };\n}\n```\n\n4. Deprecate or update the role-specific maxTokens getters:\n```javascript\n// Either remove these or update them to return maxInputTokens\nfunction getMainMaxTokens(explicitRoot = null) {\n console.warn('getMainMaxTokens is deprecated. Use getParametersForRole(\"main\") instead.');\n return getParametersForRole(\"main\", explicitRoot).maxInputTokens;\n}\n// Same for getResearchMaxTokens and getFallbackMaxTokens\n```\n\n5. Export the new functions:\n```javascript\nmodule.exports = {\n // ... existing exports\n getParametersForRole,\n getModelCapabilities\n};\n```",
|
||||
"testStrategy": "1. Unit test the updated getParametersForRole function with various configurations\n2. Verify the new getModelCapabilities function returns correct values\n3. Test with both default and custom configurations\n4. Ensure backward compatibility by checking that existing code using the old getters still works (with warnings)",
|
||||
"priority": "high",
|
||||
"dependencies": [
|
||||
82
|
||||
],
|
||||
"status": "pending",
|
||||
"subtasks": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Update config-manager.js with specific token limit fields",
|
||||
"description": "Modify the DEFAULTS object in config-manager.js to replace maxTokens with more specific token limit fields (maxInputTokens, maxOutputTokens, maxTotalTokens) and update related getter functions while maintaining backward compatibility.",
|
||||
"dependencies": [],
|
||||
"details": "1. Replace maxTokens in the DEFAULTS object with maxInputTokens, maxOutputTokens, and maxTotalTokens\n2. Update any getter functions that reference maxTokens to handle both old and new configurations\n3. Ensure backward compatibility so existing code using maxTokens continues to work\n4. Update any related documentation or comments to reflect the new token limit fields\n5. Test the changes to verify both new specific token limits and legacy maxTokens usage work correctly",
|
||||
"status": "pending"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 84,
|
||||
"title": "Implement token counting utility",
|
||||
"description": "Create a utility function to count tokens for prompts based on the model being used, primarily using tiktoken for OpenAI and Anthropic models with character-based fallbacks for other providers.",
|
||||
"details": "1. Install the tiktoken package:\n```bash\nnpm install tiktoken\n```\n\n2. Create a new file `scripts/modules/token-counter.js`:\n```javascript\nconst tiktoken = require('tiktoken');\n\n/**\n * Count tokens for a given text and model\n * @param {string} text - The text to count tokens for\n * @param {string} provider - The AI provider (e.g., 'openai', 'anthropic')\n * @param {string} modelId - The model ID\n * @returns {number} - Estimated token count\n */\nfunction countTokens(text, provider, modelId) {\n if (!text) return 0;\n \n // Convert to lowercase for case-insensitive matching\n const providerLower = provider?.toLowerCase();\n \n try {\n // OpenAI models\n if (providerLower === 'openai') {\n // Most OpenAI chat models use cl100k_base encoding\n const encoding = tiktoken.encoding_for_model(modelId) || tiktoken.get_encoding('cl100k_base');\n return encoding.encode(text).length;\n }\n \n // Anthropic models - can use cl100k_base as an approximation\n // or follow Anthropic's guidance\n if (providerLower === 'anthropic') {\n try {\n // Try to use cl100k_base as a reasonable approximation\n const encoding = tiktoken.get_encoding('cl100k_base');\n return encoding.encode(text).length;\n } catch (e) {\n // Fallback to Anthropic's character-based estimation\n return Math.ceil(text.length / 3.5); // ~3.5 chars per token for English\n }\n }\n \n // For other providers, use character-based estimation as fallback\n // Different providers may have different tokenization schemes\n return Math.ceil(text.length / 4); // General fallback estimate\n } catch (error) {\n console.warn(`Token counting error: ${error.message}. Using character-based estimate.`);\n return Math.ceil(text.length / 4); // Fallback if tiktoken fails\n }\n}\n\nmodule.exports = { countTokens };\n```\n\n3. Add tests for the token counter in `tests/token-counter.test.js`:\n```javascript\nconst { countTokens } = require('../scripts/modules/token-counter');\n\ndescribe('Token Counter', () => {\n test('counts tokens for OpenAI models', () => {\n const text = 'Hello, world! This is a test.';\n const count = countTokens(text, 'openai', 'gpt-4');\n expect(count).toBeGreaterThan(0);\n expect(typeof count).toBe('number');\n });\n \n test('counts tokens for Anthropic models', () => {\n const text = 'Hello, world! This is a test.';\n const count = countTokens(text, 'anthropic', 'claude-3-7-sonnet-20250219');\n expect(count).toBeGreaterThan(0);\n expect(typeof count).toBe('number');\n });\n \n test('handles empty text', () => {\n expect(countTokens('', 'openai', 'gpt-4')).toBe(0);\n expect(countTokens(null, 'openai', 'gpt-4')).toBe(0);\n });\n});\n```",
|
||||
"testStrategy": "1. Unit test the countTokens function with various inputs and models\n2. Compare token counts with known examples from OpenAI and Anthropic documentation\n3. Test edge cases: empty strings, very long texts, non-English texts\n4. Test fallback behavior when tiktoken fails or is not applicable",
|
||||
"priority": "high",
|
||||
"dependencies": [
|
||||
82
|
||||
],
|
||||
"status": "pending",
|
||||
"subtasks": []
|
||||
},
|
||||
{
|
||||
"id": 85,
|
||||
"title": "Update ai-services-unified.js for dynamic token limits",
|
||||
"description": "Modify the _unifiedServiceRunner function in ai-services-unified.js to use the new token counting utility and dynamically adjust output token limits based on input length.",
|
||||
"details": "1. Import the token counter in `ai-services-unified.js`:\n```javascript\nconst { countTokens } = require('./token-counter');\nconst { getParametersForRole, getModelCapabilities } = require('./config-manager');\n```\n\n2. Update the `_unifiedServiceRunner` function to implement dynamic token limit adjustment:\n```javascript\nasync function _unifiedServiceRunner({\n serviceType,\n provider,\n modelId,\n systemPrompt,\n prompt,\n temperature,\n currentRole,\n effectiveProjectRoot,\n // ... other parameters\n}) {\n // Get role parameters with new token limits\n const roleParams = getParametersForRole(currentRole, effectiveProjectRoot);\n \n // Get model capabilities\n const modelCapabilities = getModelCapabilities(provider, modelId);\n \n // Count tokens in the prompts\n const systemPromptTokens = countTokens(systemPrompt, provider, modelId);\n const userPromptTokens = countTokens(prompt, provider, modelId);\n const totalPromptTokens = systemPromptTokens + userPromptTokens;\n \n // Validate against input token limits\n if (totalPromptTokens > roleParams.maxInputTokens) {\n throw new Error(\n `Prompt (${totalPromptTokens} tokens) exceeds configured max input tokens (${roleParams.maxInputTokens}) for role '${currentRole}'.`\n );\n }\n \n // Validate against model's absolute context window\n if (modelCapabilities.contextWindowTokens && totalPromptTokens > modelCapabilities.contextWindowTokens) {\n throw new Error(\n `Prompt (${totalPromptTokens} tokens) exceeds model's context window (${modelCapabilities.contextWindowTokens}) for ${modelId}.`\n );\n }\n \n // Calculate available output tokens\n // If model has a combined context window, we need to subtract input tokens\n let availableOutputTokens = roleParams.maxOutputTokens;\n \n // If model has a context window constraint, ensure we don't exceed it\n if (modelCapabilities.contextWindowTokens) {\n const remainingContextTokens = modelCapabilities.contextWindowTokens - totalPromptTokens;\n availableOutputTokens = Math.min(availableOutputTokens, remainingContextTokens);\n }\n \n // Also respect the model's absolute max output limit\n if (modelCapabilities.maxOutputTokens) {\n availableOutputTokens = Math.min(availableOutputTokens, modelCapabilities.maxOutputTokens);\n }\n \n // Prepare API call parameters\n const callParams = {\n apiKey,\n modelId,\n maxTokens: availableOutputTokens, // Use dynamically calculated output limit\n temperature: roleParams.temperature,\n messages,\n baseUrl,\n ...(serviceType === 'generateObject' && { schema, objectName }),\n ...restApiParams\n };\n \n // Log token usage information\n console.debug(`Token usage: ${totalPromptTokens} input tokens, ${availableOutputTokens} max output tokens`);\n \n // Rest of the function remains the same...\n}\n```\n\n3. Update the error handling to provide clear messages about token limits:\n```javascript\ntry {\n // Existing code...\n} catch (error) {\n if (error.message.includes('tokens')) {\n // Token-related errors should be clearly identified\n console.error(`Token limit error: ${error.message}`);\n }\n throw error;\n}\n```",
|
||||
"testStrategy": "1. Test with prompts of various lengths to verify dynamic adjustment\n2. Test with different models to ensure model-specific limits are respected\n3. Verify error messages are clear when limits are exceeded\n4. Test edge cases: very short prompts, prompts near the limit\n5. Integration test with actual API calls to verify the calculated limits work in practice",
|
||||
"priority": "medium",
|
||||
"dependencies": [
|
||||
83,
|
||||
84
|
||||
],
|
||||
"status": "pending",
|
||||
"subtasks": []
|
||||
},
|
||||
{
|
||||
"id": 86,
|
||||
"title": "Update .taskmasterconfig schema and user guide",
|
||||
"description": "Create a migration guide for users to update their .taskmasterconfig files and document the new token limit configuration options.",
|
||||
"details": "1. Create a migration script or guide for users to update their existing `.taskmasterconfig` files:\n\n```javascript\n// Example migration snippet for .taskmasterconfig\n{\n \"main\": {\n // Before:\n // \"maxTokens\": 16000,\n \n // After:\n \"maxInputTokens\": 16000,\n \"maxOutputTokens\": 4000,\n \"temperature\": 0.7\n },\n \"research\": {\n \"maxInputTokens\": 16000,\n \"maxOutputTokens\": 4000,\n \"temperature\": 0.7\n },\n \"fallback\": {\n \"maxInputTokens\": 8000,\n \"maxOutputTokens\": 2000,\n \"temperature\": 0.7\n }\n}\n```\n\n2. Update the user documentation to explain the new token limit fields:\n\n```markdown\n# Token Limit Configuration\n\nTask Master now provides more granular control over token limits with separate settings for input and output tokens:\n\n- `maxInputTokens`: Maximum number of tokens allowed in the input prompt (system prompt + user prompt)\n- `maxOutputTokens`: Maximum number of tokens the model should generate in its response\n\n## Benefits\n\n- More precise control over token usage\n- Better cost management\n- Reduced likelihood of hitting model context limits\n- Dynamic adjustment to maximize output space based on input length\n\n## Migration from Previous Versions\n\nIf you're upgrading from a previous version, you'll need to update your `.taskmasterconfig` file:\n\n1. Replace the single `maxTokens` field with separate `maxInputTokens` and `maxOutputTokens` fields\n2. Recommended starting values:\n - Set `maxInputTokens` to your previous `maxTokens` value\n - Set `maxOutputTokens` to approximately 1/4 of your model's context window\n\n## Example Configuration\n\n```json\n{\n \"main\": {\n \"maxInputTokens\": 16000,\n \"maxOutputTokens\": 4000,\n \"temperature\": 0.7\n }\n}\n```\n```\n\n3. Update the schema validation in `config-manager.js` to validate the new fields:\n\n```javascript\nfunction _validateConfig(config) {\n // ... existing validation\n \n // Validate token limits for each role\n ['main', 'research', 'fallback'].forEach(role => {\n if (config[role]) {\n // Check if old maxTokens is present and warn about migration\n if (config[role].maxTokens !== undefined) {\n console.warn(`Warning: 'maxTokens' in ${role} role is deprecated. Please use 'maxInputTokens' and 'maxOutputTokens' instead.`);\n }\n \n // Validate new token limit fields\n if (config[role].maxInputTokens !== undefined && (!Number.isInteger(config[role].maxInputTokens) || config[role].maxInputTokens <= 0)) {\n throw new Error(`Invalid maxInputTokens for ${role} role: must be a positive integer`);\n }\n \n if (config[role].maxOutputTokens !== undefined && (!Number.isInteger(config[role].maxOutputTokens) || config[role].maxOutputTokens <= 0)) {\n throw new Error(`Invalid maxOutputTokens for ${role} role: must be a positive integer`);\n }\n }\n });\n \n return config;\n}\n```",
|
||||
"testStrategy": "1. Verify documentation is clear and provides migration steps\n2. Test the validation logic with various config formats\n3. Test backward compatibility with old config format\n4. Ensure error messages are helpful when validation fails",
|
||||
"priority": "medium",
|
||||
"dependencies": [
|
||||
83
|
||||
],
|
||||
"status": "pending",
|
||||
"subtasks": []
|
||||
},
|
||||
{
|
||||
"id": 87,
|
||||
"title": "Implement validation and error handling",
|
||||
"description": "Add comprehensive validation and error handling for token limits throughout the system, including helpful error messages and graceful fallbacks.",
|
||||
"details": "1. Add validation when loading models in `config-manager.js`:\n```javascript\nfunction _validateModelMap(modelMap) {\n // Validate each provider's models\n Object.entries(modelMap).forEach(([provider, models]) => {\n models.forEach(model => {\n // Check for required token limit fields\n if (!model.contextWindowTokens) {\n console.warn(`Warning: Model ${model.id} from ${provider} is missing contextWindowTokens field`);\n }\n if (!model.maxOutputTokens) {\n console.warn(`Warning: Model ${model.id} from ${provider} is missing maxOutputTokens field`);\n }\n });\n });\n return modelMap;\n}\n```\n\n2. Add validation when setting up a model in the CLI:\n```javascript\nfunction validateModelConfig(modelConfig, modelCapabilities) {\n const issues = [];\n \n // Check if input tokens exceed model's context window\n if (modelConfig.maxInputTokens > modelCapabilities.contextWindowTokens) {\n issues.push(`maxInputTokens (${modelConfig.maxInputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);\n }\n \n // Check if output tokens exceed model's maximum\n if (modelConfig.maxOutputTokens > modelCapabilities.maxOutputTokens) {\n issues.push(`maxOutputTokens (${modelConfig.maxOutputTokens}) exceeds model's maximum output tokens (${modelCapabilities.maxOutputTokens})`);\n }\n \n // Check if combined tokens exceed context window\n if (modelConfig.maxInputTokens + modelConfig.maxOutputTokens > modelCapabilities.contextWindowTokens) {\n issues.push(`Combined maxInputTokens and maxOutputTokens (${modelConfig.maxInputTokens + modelConfig.maxOutputTokens}) exceeds model's context window (${modelCapabilities.contextWindowTokens})`);\n }\n \n return issues;\n}\n```\n\n3. Add graceful fallbacks in `ai-services-unified.js`:\n```javascript\n// Fallback for missing token limits\nif (!roleParams.maxInputTokens) {\n console.warn(`Warning: maxInputTokens not specified for role '${currentRole}'. Using default value.`);\n roleParams.maxInputTokens = 8000; // Reasonable default\n}\n\nif (!roleParams.maxOutputTokens) {\n console.warn(`Warning: maxOutputTokens not specified for role '${currentRole}'. Using default value.`);\n roleParams.maxOutputTokens = 2000; // Reasonable default\n}\n\n// Fallback for missing model capabilities\nif (!modelCapabilities.contextWindowTokens) {\n console.warn(`Warning: contextWindowTokens not specified for model ${modelId}. Using conservative estimate.`);\n modelCapabilities.contextWindowTokens = roleParams.maxInputTokens + roleParams.maxOutputTokens;\n}\n\nif (!modelCapabilities.maxOutputTokens) {\n console.warn(`Warning: maxOutputTokens not specified for model ${modelId}. Using role configuration.`);\n modelCapabilities.maxOutputTokens = roleParams.maxOutputTokens;\n}\n```\n\n4. Add detailed logging for token usage:\n```javascript\nfunction logTokenUsage(provider, modelId, inputTokens, outputTokens, role) {\n const inputCost = calculateTokenCost(provider, modelId, 'input', inputTokens);\n const outputCost = calculateTokenCost(provider, modelId, 'output', outputTokens);\n \n console.info(`Token usage for ${role} role with ${provider}/${modelId}:`);\n console.info(`- Input: ${inputTokens.toLocaleString()} tokens ($${inputCost.toFixed(6)})`);\n console.info(`- Output: ${outputTokens.toLocaleString()} tokens ($${outputCost.toFixed(6)})`);\n console.info(`- Total cost: $${(inputCost + outputCost).toFixed(6)}`);\n console.info(`- Available output tokens: ${availableOutputTokens.toLocaleString()}`);\n}\n```\n\n5. Add a helper function to suggest configuration improvements:\n```javascript\nfunction suggestTokenConfigImprovements(roleParams, modelCapabilities, promptTokens) {\n const suggestions = [];\n \n // If prompt is using less than 50% of allowed input\n if (promptTokens < roleParams.maxInputTokens * 0.5) {\n suggestions.push(`Consider reducing maxInputTokens from ${roleParams.maxInputTokens} to save on potential costs`);\n }\n \n // If output tokens are very limited due to large input\n const availableOutput = Math.min(\n roleParams.maxOutputTokens,\n modelCapabilities.contextWindowTokens - promptTokens\n );\n \n if (availableOutput < roleParams.maxOutputTokens * 0.5) {\n suggestions.push(`Available output tokens (${availableOutput}) are significantly less than configured maxOutputTokens (${roleParams.maxOutputTokens}) due to large input`);\n }\n \n return suggestions;\n}\n```",
|
||||
"testStrategy": "1. Test validation functions with valid and invalid configurations\n2. Verify fallback behavior works correctly when configuration is missing\n3. Test error messages are clear and actionable\n4. Test logging functions provide useful information\n5. Verify suggestion logic provides helpful recommendations",
|
||||
"priority": "low",
|
||||
"dependencies": [
|
||||
85
|
||||
],
|
||||
"status": "pending",
|
||||
"subtasks": []
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user