chore: run format
This commit is contained in:
Ralph Khreish
@@ -3,234 +3,245 @@ import { join } from 'path';
|
||||
import chalk from 'chalk';
|
||||
|
||||
export class ErrorHandler {
|
||||
constructor(logger) {
|
||||
this.logger = logger;
|
||||
this.errors = [];
|
||||
this.warnings = [];
|
||||
}
|
||||
constructor(logger) {
|
||||
this.logger = logger;
|
||||
this.errors = [];
|
||||
this.warnings = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle and categorize errors
|
||||
*/
|
||||
handleError(error, context = {}) {
|
||||
const errorInfo = {
|
||||
timestamp: new Date().toISOString(),
|
||||
message: error.message || 'Unknown error',
|
||||
stack: error.stack,
|
||||
context,
|
||||
type: this.categorizeError(error)
|
||||
};
|
||||
/**
|
||||
* Handle and categorize errors
|
||||
*/
|
||||
handleError(error, context = {}) {
|
||||
const errorInfo = {
|
||||
timestamp: new Date().toISOString(),
|
||||
message: error.message || 'Unknown error',
|
||||
stack: error.stack,
|
||||
context,
|
||||
type: this.categorizeError(error)
|
||||
};
|
||||
|
||||
this.errors.push(errorInfo);
|
||||
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
|
||||
this.errors.push(errorInfo);
|
||||
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
|
||||
|
||||
if (context.critical) {
|
||||
throw error;
|
||||
}
|
||||
if (context.critical) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return errorInfo;
|
||||
}
|
||||
return errorInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a warning
|
||||
*/
|
||||
addWarning(message, context = {}) {
|
||||
const warning = {
|
||||
timestamp: new Date().toISOString(),
|
||||
message,
|
||||
context
|
||||
};
|
||||
/**
|
||||
* Add a warning
|
||||
*/
|
||||
addWarning(message, context = {}) {
|
||||
const warning = {
|
||||
timestamp: new Date().toISOString(),
|
||||
message,
|
||||
context
|
||||
};
|
||||
|
||||
this.warnings.push(warning);
|
||||
this.logger.warning(message);
|
||||
}
|
||||
this.warnings.push(warning);
|
||||
this.logger.warning(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize error types
|
||||
*/
|
||||
categorizeError(error) {
|
||||
const message = error.message.toLowerCase();
|
||||
/**
|
||||
* Categorize error types
|
||||
*/
|
||||
categorizeError(error) {
|
||||
const message = error.message.toLowerCase();
|
||||
|
||||
if (message.includes('command not found') || message.includes('not found')) {
|
||||
return 'DEPENDENCY_ERROR';
|
||||
}
|
||||
if (message.includes('permission') || message.includes('access denied')) {
|
||||
return 'PERMISSION_ERROR';
|
||||
}
|
||||
if (message.includes('timeout')) {
|
||||
return 'TIMEOUT_ERROR';
|
||||
}
|
||||
if (message.includes('api') || message.includes('rate limit')) {
|
||||
return 'API_ERROR';
|
||||
}
|
||||
if (message.includes('json') || message.includes('parse')) {
|
||||
return 'PARSE_ERROR';
|
||||
}
|
||||
if (message.includes('file') || message.includes('directory')) {
|
||||
return 'FILE_ERROR';
|
||||
}
|
||||
if (
|
||||
message.includes('command not found') ||
|
||||
message.includes('not found')
|
||||
) {
|
||||
return 'DEPENDENCY_ERROR';
|
||||
}
|
||||
if (message.includes('permission') || message.includes('access denied')) {
|
||||
return 'PERMISSION_ERROR';
|
||||
}
|
||||
if (message.includes('timeout')) {
|
||||
return 'TIMEOUT_ERROR';
|
||||
}
|
||||
if (message.includes('api') || message.includes('rate limit')) {
|
||||
return 'API_ERROR';
|
||||
}
|
||||
if (message.includes('json') || message.includes('parse')) {
|
||||
return 'PARSE_ERROR';
|
||||
}
|
||||
if (message.includes('file') || message.includes('directory')) {
|
||||
return 'FILE_ERROR';
|
||||
}
|
||||
|
||||
return 'GENERAL_ERROR';
|
||||
}
|
||||
return 'GENERAL_ERROR';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get error summary
|
||||
*/
|
||||
getSummary() {
|
||||
const errorsByType = {};
|
||||
|
||||
this.errors.forEach(error => {
|
||||
if (!errorsByType[error.type]) {
|
||||
errorsByType[error.type] = [];
|
||||
}
|
||||
errorsByType[error.type].push(error);
|
||||
});
|
||||
/**
|
||||
* Get error summary
|
||||
*/
|
||||
getSummary() {
|
||||
const errorsByType = {};
|
||||
|
||||
return {
|
||||
totalErrors: this.errors.length,
|
||||
totalWarnings: this.warnings.length,
|
||||
errorsByType,
|
||||
criticalErrors: this.errors.filter(e => e.context.critical),
|
||||
recentErrors: this.errors.slice(-5)
|
||||
};
|
||||
}
|
||||
this.errors.forEach((error) => {
|
||||
if (!errorsByType[error.type]) {
|
||||
errorsByType[error.type] = [];
|
||||
}
|
||||
errorsByType[error.type].push(error);
|
||||
});
|
||||
|
||||
/**
|
||||
* Generate error report
|
||||
*/
|
||||
generateReport(outputPath) {
|
||||
const summary = this.getSummary();
|
||||
const report = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
summary: {
|
||||
totalErrors: summary.totalErrors,
|
||||
totalWarnings: summary.totalWarnings,
|
||||
errorTypes: Object.keys(summary.errorsByType)
|
||||
},
|
||||
errors: this.errors,
|
||||
warnings: this.warnings,
|
||||
recommendations: this.generateRecommendations(summary)
|
||||
};
|
||||
return {
|
||||
totalErrors: this.errors.length,
|
||||
totalWarnings: this.warnings.length,
|
||||
errorsByType,
|
||||
criticalErrors: this.errors.filter((e) => e.context.critical),
|
||||
recentErrors: this.errors.slice(-5)
|
||||
};
|
||||
}
|
||||
|
||||
writeFileSync(outputPath, JSON.stringify(report, null, 2));
|
||||
return report;
|
||||
}
|
||||
/**
|
||||
* Generate error report
|
||||
*/
|
||||
generateReport(outputPath) {
|
||||
const summary = this.getSummary();
|
||||
const report = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
summary: {
|
||||
totalErrors: summary.totalErrors,
|
||||
totalWarnings: summary.totalWarnings,
|
||||
errorTypes: Object.keys(summary.errorsByType)
|
||||
},
|
||||
errors: this.errors,
|
||||
warnings: this.warnings,
|
||||
recommendations: this.generateRecommendations(summary)
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate recommendations based on errors
|
||||
*/
|
||||
generateRecommendations(summary) {
|
||||
const recommendations = [];
|
||||
writeFileSync(outputPath, JSON.stringify(report, null, 2));
|
||||
return report;
|
||||
}
|
||||
|
||||
if (summary.errorsByType.DEPENDENCY_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'DEPENDENCY',
|
||||
message: 'Install missing dependencies using npm install or check PATH',
|
||||
errors: summary.errorsByType.DEPENDENCY_ERROR.length
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Generate recommendations based on errors
|
||||
*/
|
||||
generateRecommendations(summary) {
|
||||
const recommendations = [];
|
||||
|
||||
if (summary.errorsByType.PERMISSION_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'PERMISSION',
|
||||
message: 'Check file permissions or run with appropriate privileges',
|
||||
errors: summary.errorsByType.PERMISSION_ERROR.length
|
||||
});
|
||||
}
|
||||
if (summary.errorsByType.DEPENDENCY_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'DEPENDENCY',
|
||||
message: 'Install missing dependencies using npm install or check PATH',
|
||||
errors: summary.errorsByType.DEPENDENCY_ERROR.length
|
||||
});
|
||||
}
|
||||
|
||||
if (summary.errorsByType.API_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'API',
|
||||
message: 'Check API keys, rate limits, or network connectivity',
|
||||
errors: summary.errorsByType.API_ERROR.length
|
||||
});
|
||||
}
|
||||
if (summary.errorsByType.PERMISSION_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'PERMISSION',
|
||||
message: 'Check file permissions or run with appropriate privileges',
|
||||
errors: summary.errorsByType.PERMISSION_ERROR.length
|
||||
});
|
||||
}
|
||||
|
||||
if (summary.errorsByType.TIMEOUT_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'TIMEOUT',
|
||||
message: 'Consider increasing timeout values or optimizing slow operations',
|
||||
errors: summary.errorsByType.TIMEOUT_ERROR.length
|
||||
});
|
||||
}
|
||||
if (summary.errorsByType.API_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'API',
|
||||
message: 'Check API keys, rate limits, or network connectivity',
|
||||
errors: summary.errorsByType.API_ERROR.length
|
||||
});
|
||||
}
|
||||
|
||||
return recommendations;
|
||||
}
|
||||
if (summary.errorsByType.TIMEOUT_ERROR) {
|
||||
recommendations.push({
|
||||
type: 'TIMEOUT',
|
||||
message:
|
||||
'Consider increasing timeout values or optimizing slow operations',
|
||||
errors: summary.errorsByType.TIMEOUT_ERROR.length
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Display error summary in console
|
||||
*/
|
||||
displaySummary() {
|
||||
const summary = this.getSummary();
|
||||
return recommendations;
|
||||
}
|
||||
|
||||
if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
|
||||
console.log(chalk.green('✅ No errors or warnings detected'));
|
||||
return;
|
||||
}
|
||||
/**
|
||||
* Display error summary in console
|
||||
*/
|
||||
displaySummary() {
|
||||
const summary = this.getSummary();
|
||||
|
||||
console.log(chalk.red.bold(`\n🚨 Error Summary:`));
|
||||
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
|
||||
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
|
||||
if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
|
||||
console.log(chalk.green('✅ No errors or warnings detected'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (summary.totalErrors > 0) {
|
||||
console.log(chalk.red.bold('\n Error Types:'));
|
||||
Object.entries(summary.errorsByType).forEach(([type, errors]) => {
|
||||
console.log(chalk.red(` - ${type}: ${errors.length}`));
|
||||
});
|
||||
console.log(chalk.red.bold(`\n🚨 Error Summary:`));
|
||||
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
|
||||
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
|
||||
|
||||
if (summary.criticalErrors.length > 0) {
|
||||
console.log(chalk.red.bold(`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`));
|
||||
summary.criticalErrors.forEach(error => {
|
||||
console.log(chalk.red(` - ${error.message}`));
|
||||
});
|
||||
}
|
||||
}
|
||||
if (summary.totalErrors > 0) {
|
||||
console.log(chalk.red.bold('\n Error Types:'));
|
||||
Object.entries(summary.errorsByType).forEach(([type, errors]) => {
|
||||
console.log(chalk.red(` - ${type}: ${errors.length}`));
|
||||
});
|
||||
|
||||
const recommendations = this.generateRecommendations(summary);
|
||||
if (recommendations.length > 0) {
|
||||
console.log(chalk.yellow.bold('\n💡 Recommendations:'));
|
||||
recommendations.forEach(rec => {
|
||||
console.log(chalk.yellow(` - ${rec.message}`));
|
||||
});
|
||||
}
|
||||
}
|
||||
if (summary.criticalErrors.length > 0) {
|
||||
console.log(
|
||||
chalk.red.bold(
|
||||
`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`
|
||||
)
|
||||
);
|
||||
summary.criticalErrors.forEach((error) => {
|
||||
console.log(chalk.red(` - ${error.message}`));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all errors and warnings
|
||||
*/
|
||||
clear() {
|
||||
this.errors = [];
|
||||
this.warnings = [];
|
||||
}
|
||||
const recommendations = this.generateRecommendations(summary);
|
||||
if (recommendations.length > 0) {
|
||||
console.log(chalk.yellow.bold('\n💡 Recommendations:'));
|
||||
recommendations.forEach((rec) => {
|
||||
console.log(chalk.yellow(` - ${rec.message}`));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all errors and warnings
|
||||
*/
|
||||
clear() {
|
||||
this.errors = [];
|
||||
this.warnings = [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Global error handler for uncaught exceptions
|
||||
*/
|
||||
export function setupGlobalErrorHandlers(errorHandler, logger) {
|
||||
process.on('uncaughtException', (error) => {
|
||||
logger.error(`Uncaught Exception: ${error.message}`);
|
||||
errorHandler.handleError(error, { critical: true, source: 'uncaughtException' });
|
||||
process.exit(1);
|
||||
});
|
||||
process.on('uncaughtException', (error) => {
|
||||
logger.error(`Uncaught Exception: ${error.message}`);
|
||||
errorHandler.handleError(error, {
|
||||
critical: true,
|
||||
source: 'uncaughtException'
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
|
||||
errorHandler.handleError(new Error(String(reason)), {
|
||||
critical: false,
|
||||
source: 'unhandledRejection'
|
||||
});
|
||||
});
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
|
||||
errorHandler.handleError(new Error(String(reason)), {
|
||||
critical: false,
|
||||
source: 'unhandledRejection'
|
||||
});
|
||||
});
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
logger.info('\nReceived SIGINT, shutting down gracefully...');
|
||||
errorHandler.displaySummary();
|
||||
process.exit(130);
|
||||
});
|
||||
process.on('SIGINT', () => {
|
||||
logger.info('\nReceived SIGINT, shutting down gracefully...');
|
||||
errorHandler.displaySummary();
|
||||
process.exit(130);
|
||||
});
|
||||
|
||||
process.on('SIGTERM', () => {
|
||||
logger.info('\nReceived SIGTERM, shutting down...');
|
||||
errorHandler.displaySummary();
|
||||
process.exit(143);
|
||||
});
|
||||
}
|
||||
process.on('SIGTERM', () => {
|
||||
logger.info('\nReceived SIGTERM, shutting down...');
|
||||
errorHandler.displaySummary();
|
||||
process.exit(143);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,56 +2,58 @@ import { readFileSync } from 'fs';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export class LLMAnalyzer {
|
||||
constructor(config, logger) {
|
||||
this.config = config;
|
||||
this.logger = logger;
|
||||
this.apiKey = process.env.ANTHROPIC_API_KEY;
|
||||
this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
|
||||
}
|
||||
constructor(config, logger) {
|
||||
this.config = config;
|
||||
this.logger = logger;
|
||||
this.apiKey = process.env.ANTHROPIC_API_KEY;
|
||||
this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
|
||||
}
|
||||
|
||||
async analyzeLog(logFile, providerSummaryFile = null) {
|
||||
if (!this.config.llmAnalysis.enabled) {
|
||||
this.logger.info('LLM analysis is disabled in configuration');
|
||||
return null;
|
||||
}
|
||||
async analyzeLog(logFile, providerSummaryFile = null) {
|
||||
if (!this.config.llmAnalysis.enabled) {
|
||||
this.logger.info('LLM analysis is disabled in configuration');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!this.apiKey) {
|
||||
this.logger.error('ANTHROPIC_API_KEY not found in environment');
|
||||
return null;
|
||||
}
|
||||
if (!this.apiKey) {
|
||||
this.logger.error('ANTHROPIC_API_KEY not found in environment');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const logContent = readFileSync(logFile, 'utf8');
|
||||
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
|
||||
try {
|
||||
const logContent = readFileSync(logFile, 'utf8');
|
||||
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
|
||||
|
||||
const response = await this.callLLM(prompt);
|
||||
const analysis = this.parseResponse(response);
|
||||
|
||||
// Calculate and log cost
|
||||
if (response.usage) {
|
||||
const cost = this.calculateCost(response.usage);
|
||||
this.logger.addCost(cost);
|
||||
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
|
||||
}
|
||||
const response = await this.callLLM(prompt);
|
||||
const analysis = this.parseResponse(response);
|
||||
|
||||
return analysis;
|
||||
} catch (error) {
|
||||
this.logger.error(`LLM analysis failed: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Calculate and log cost
|
||||
if (response.usage) {
|
||||
const cost = this.calculateCost(response.usage);
|
||||
this.logger.addCost(cost);
|
||||
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
|
||||
}
|
||||
|
||||
buildAnalysisPrompt(logContent, providerSummaryFile) {
|
||||
let providerSummary = '';
|
||||
if (providerSummaryFile) {
|
||||
try {
|
||||
providerSummary = readFileSync(providerSummaryFile, 'utf8');
|
||||
} catch (error) {
|
||||
this.logger.warning(`Could not read provider summary file: ${error.message}`);
|
||||
}
|
||||
}
|
||||
return analysis;
|
||||
} catch (error) {
|
||||
this.logger.error(`LLM analysis failed: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
|
||||
buildAnalysisPrompt(logContent, providerSummaryFile) {
|
||||
let providerSummary = '';
|
||||
if (providerSummaryFile) {
|
||||
try {
|
||||
providerSummary = readFileSync(providerSummaryFile, 'utf8');
|
||||
} catch (error) {
|
||||
this.logger.warning(
|
||||
`Could not read provider summary file: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
|
||||
|
||||
Your goal is to:
|
||||
1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files).
|
||||
@@ -88,81 +90,82 @@ Return your analysis **strictly** in the following JSON format. Do not include a
|
||||
Here is the main log content:
|
||||
|
||||
${logContent}`;
|
||||
}
|
||||
}
|
||||
|
||||
async callLLM(prompt) {
|
||||
const payload = {
|
||||
model: this.config.llmAnalysis.model,
|
||||
max_tokens: this.config.llmAnalysis.maxTokens,
|
||||
messages: [
|
||||
{ role: 'user', content: prompt }
|
||||
]
|
||||
};
|
||||
async callLLM(prompt) {
|
||||
const payload = {
|
||||
model: this.config.llmAnalysis.model,
|
||||
max_tokens: this.config.llmAnalysis.maxTokens,
|
||||
messages: [{ role: 'user', content: prompt }]
|
||||
};
|
||||
|
||||
const response = await fetch(this.apiEndpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': this.apiKey,
|
||||
'anthropic-version': '2023-06-01'
|
||||
},
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
const response = await fetch(this.apiEndpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': this.apiKey,
|
||||
'anthropic-version': '2023-06-01'
|
||||
},
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`LLM API call failed: ${response.status} - ${error}`);
|
||||
}
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`LLM API call failed: ${response.status} - ${error}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
parseResponse(response) {
|
||||
try {
|
||||
const content = response.content[0].text;
|
||||
const jsonStart = content.indexOf('{');
|
||||
const jsonEnd = content.lastIndexOf('}');
|
||||
|
||||
if (jsonStart === -1 || jsonEnd === -1) {
|
||||
throw new Error('No JSON found in response');
|
||||
}
|
||||
parseResponse(response) {
|
||||
try {
|
||||
const content = response.content[0].text;
|
||||
const jsonStart = content.indexOf('{');
|
||||
const jsonEnd = content.lastIndexOf('}');
|
||||
|
||||
const jsonString = content.substring(jsonStart, jsonEnd + 1);
|
||||
return JSON.parse(jsonString);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to parse LLM response: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (jsonStart === -1 || jsonEnd === -1) {
|
||||
throw new Error('No JSON found in response');
|
||||
}
|
||||
|
||||
calculateCost(usage) {
|
||||
const modelCosts = {
|
||||
'claude-3-7-sonnet-20250219': {
|
||||
input: 3.00, // per 1M tokens
|
||||
output: 15.00 // per 1M tokens
|
||||
}
|
||||
};
|
||||
const jsonString = content.substring(jsonStart, jsonEnd + 1);
|
||||
return JSON.parse(jsonString);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to parse LLM response: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const costs = modelCosts[this.config.llmAnalysis.model] || { input: 0, output: 0 };
|
||||
const inputCost = (usage.input_tokens / 1000000) * costs.input;
|
||||
const outputCost = (usage.output_tokens / 1000000) * costs.output;
|
||||
|
||||
return inputCost + outputCost;
|
||||
}
|
||||
calculateCost(usage) {
|
||||
const modelCosts = {
|
||||
'claude-3-7-sonnet-20250219': {
|
||||
input: 3.0, // per 1M tokens
|
||||
output: 15.0 // per 1M tokens
|
||||
}
|
||||
};
|
||||
|
||||
formatReport(analysis) {
|
||||
if (!analysis) return null;
|
||||
const costs = modelCosts[this.config.llmAnalysis.model] || {
|
||||
input: 0,
|
||||
output: 0
|
||||
};
|
||||
const inputCost = (usage.input_tokens / 1000000) * costs.input;
|
||||
const outputCost = (usage.output_tokens / 1000000) * costs.output;
|
||||
|
||||
const report = {
|
||||
title: 'TASKMASTER E2E Test Analysis Report',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: analysis.overall_status,
|
||||
summary: analysis.llm_summary_points,
|
||||
verifiedSteps: analysis.verified_steps,
|
||||
providerComparison: analysis.provider_add_task_comparison,
|
||||
issues: analysis.detected_issues
|
||||
};
|
||||
return inputCost + outputCost;
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
}
|
||||
formatReport(analysis) {
|
||||
if (!analysis) return null;
|
||||
|
||||
const report = {
|
||||
title: 'TASKMASTER E2E Test Analysis Report',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: analysis.overall_status,
|
||||
summary: analysis.llm_summary_points,
|
||||
verifiedSteps: analysis.verified_steps,
|
||||
providerComparison: analysis.provider_add_task_comparison,
|
||||
issues: analysis.detected_issues
|
||||
};
|
||||
|
||||
return report;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,122 +3,129 @@ import { join } from 'path';
|
||||
import chalk from 'chalk';
|
||||
|
||||
export class TestLogger {
|
||||
constructor(logDir, testRunId) {
|
||||
this.logDir = logDir;
|
||||
this.testRunId = testRunId;
|
||||
this.startTime = Date.now();
|
||||
this.stepCount = 0;
|
||||
this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
|
||||
this.logBuffer = [];
|
||||
this.totalCost = 0;
|
||||
constructor(logDir, testRunId) {
|
||||
this.logDir = logDir;
|
||||
this.testRunId = testRunId;
|
||||
this.startTime = Date.now();
|
||||
this.stepCount = 0;
|
||||
this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
|
||||
this.logBuffer = [];
|
||||
this.totalCost = 0;
|
||||
|
||||
// Ensure log directory exists
|
||||
if (!existsSync(logDir)) {
|
||||
mkdirSync(logDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
// Ensure log directory exists
|
||||
if (!existsSync(logDir)) {
|
||||
mkdirSync(logDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
formatDuration(milliseconds) {
|
||||
const totalSeconds = Math.floor(milliseconds / 1000);
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
|
||||
}
|
||||
formatDuration(milliseconds) {
|
||||
const totalSeconds = Math.floor(milliseconds / 1000);
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
|
||||
}
|
||||
|
||||
getElapsedTime() {
|
||||
return this.formatDuration(Date.now() - this.startTime);
|
||||
}
|
||||
getElapsedTime() {
|
||||
return this.formatDuration(Date.now() - this.startTime);
|
||||
}
|
||||
|
||||
formatLogEntry(level, message) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const elapsed = this.getElapsedTime();
|
||||
return `[${level}] [${elapsed}] ${timestamp} ${message}`;
|
||||
}
|
||||
formatLogEntry(level, message) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const elapsed = this.getElapsedTime();
|
||||
return `[${level}] [${elapsed}] ${timestamp} ${message}`;
|
||||
}
|
||||
|
||||
log(level, message, options = {}) {
|
||||
const formattedMessage = this.formatLogEntry(level, message);
|
||||
|
||||
// Add to buffer
|
||||
this.logBuffer.push(formattedMessage);
|
||||
|
||||
// Console output with colors
|
||||
let coloredMessage = formattedMessage;
|
||||
switch (level) {
|
||||
case 'INFO':
|
||||
coloredMessage = chalk.blue(formattedMessage);
|
||||
break;
|
||||
case 'SUCCESS':
|
||||
coloredMessage = chalk.green(formattedMessage);
|
||||
break;
|
||||
case 'ERROR':
|
||||
coloredMessage = chalk.red(formattedMessage);
|
||||
break;
|
||||
case 'WARNING':
|
||||
coloredMessage = chalk.yellow(formattedMessage);
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(coloredMessage);
|
||||
|
||||
// Write to file if immediate flush requested
|
||||
if (options.flush) {
|
||||
this.flush();
|
||||
}
|
||||
}
|
||||
log(level, message, options = {}) {
|
||||
const formattedMessage = this.formatLogEntry(level, message);
|
||||
|
||||
info(message) {
|
||||
this.log('INFO', message);
|
||||
}
|
||||
// Add to buffer
|
||||
this.logBuffer.push(formattedMessage);
|
||||
|
||||
success(message) {
|
||||
this.log('SUCCESS', message);
|
||||
}
|
||||
// Console output with colors
|
||||
let coloredMessage = formattedMessage;
|
||||
switch (level) {
|
||||
case 'INFO':
|
||||
coloredMessage = chalk.blue(formattedMessage);
|
||||
break;
|
||||
case 'SUCCESS':
|
||||
coloredMessage = chalk.green(formattedMessage);
|
||||
break;
|
||||
case 'ERROR':
|
||||
coloredMessage = chalk.red(formattedMessage);
|
||||
break;
|
||||
case 'WARNING':
|
||||
coloredMessage = chalk.yellow(formattedMessage);
|
||||
break;
|
||||
}
|
||||
|
||||
error(message) {
|
||||
this.log('ERROR', message);
|
||||
}
|
||||
console.log(coloredMessage);
|
||||
|
||||
warning(message) {
|
||||
this.log('WARNING', message);
|
||||
}
|
||||
// Write to file if immediate flush requested
|
||||
if (options.flush) {
|
||||
this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
step(message) {
|
||||
this.stepCount++;
|
||||
const separator = '='.repeat(45);
|
||||
this.log('STEP', `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`);
|
||||
}
|
||||
info(message) {
|
||||
this.log('INFO', message);
|
||||
}
|
||||
|
||||
addCost(cost) {
|
||||
if (typeof cost === 'number' && !isNaN(cost)) {
|
||||
this.totalCost += cost;
|
||||
}
|
||||
}
|
||||
success(message) {
|
||||
this.log('SUCCESS', message);
|
||||
}
|
||||
|
||||
extractAndAddCost(output) {
|
||||
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
|
||||
let match;
|
||||
while ((match = costRegex.exec(output)) !== null) {
|
||||
const cost = parseFloat(match[1]);
|
||||
this.addCost(cost);
|
||||
}
|
||||
}
|
||||
error(message) {
|
||||
this.log('ERROR', message);
|
||||
}
|
||||
|
||||
flush() {
|
||||
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
|
||||
}
|
||||
warning(message) {
|
||||
this.log('WARNING', message);
|
||||
}
|
||||
|
||||
getSummary() {
|
||||
const duration = this.formatDuration(Date.now() - this.startTime);
|
||||
const successCount = this.logBuffer.filter(line => line.includes('[SUCCESS]')).length;
|
||||
const errorCount = this.logBuffer.filter(line => line.includes('[ERROR]')).length;
|
||||
|
||||
return {
|
||||
duration,
|
||||
totalSteps: this.stepCount,
|
||||
successCount,
|
||||
errorCount,
|
||||
totalCost: this.totalCost.toFixed(6),
|
||||
logFile: this.logFile
|
||||
};
|
||||
}
|
||||
}
|
||||
step(message) {
|
||||
this.stepCount++;
|
||||
const separator = '='.repeat(45);
|
||||
this.log(
|
||||
'STEP',
|
||||
`\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`
|
||||
);
|
||||
}
|
||||
|
||||
addCost(cost) {
|
||||
if (typeof cost === 'number' && !isNaN(cost)) {
|
||||
this.totalCost += cost;
|
||||
}
|
||||
}
|
||||
|
||||
extractAndAddCost(output) {
|
||||
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
|
||||
let match;
|
||||
while ((match = costRegex.exec(output)) !== null) {
|
||||
const cost = parseFloat(match[1]);
|
||||
this.addCost(cost);
|
||||
}
|
||||
}
|
||||
|
||||
flush() {
|
||||
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
|
||||
}
|
||||
|
||||
getSummary() {
|
||||
const duration = this.formatDuration(Date.now() - this.startTime);
|
||||
const successCount = this.logBuffer.filter((line) =>
|
||||
line.includes('[SUCCESS]')
|
||||
).length;
|
||||
const errorCount = this.logBuffer.filter((line) =>
|
||||
line.includes('[ERROR]')
|
||||
).length;
|
||||
|
||||
return {
|
||||
duration,
|
||||
totalSteps: this.stepCount,
|
||||
successCount,
|
||||
errorCount,
|
||||
totalCost: this.totalCost.toFixed(6),
|
||||
logFile: this.logFile
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,185 +3,190 @@ import { readFileSync, existsSync, copyFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
export class TestHelpers {
|
||||
constructor(logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
constructor(logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a command and return output
|
||||
* @param {string} command - Command to execute
|
||||
* @param {string[]} args - Command arguments
|
||||
* @param {Object} options - Execution options
|
||||
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
|
||||
*/
|
||||
async executeCommand(command, args = [], options = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const spawnOptions = {
|
||||
cwd: options.cwd || process.cwd(),
|
||||
env: { ...process.env, ...options.env },
|
||||
shell: true
|
||||
};
|
||||
/**
|
||||
* Execute a command and return output
|
||||
* @param {string} command - Command to execute
|
||||
* @param {string[]} args - Command arguments
|
||||
* @param {Object} options - Execution options
|
||||
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
|
||||
*/
|
||||
async executeCommand(command, args = [], options = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const spawnOptions = {
|
||||
cwd: options.cwd || process.cwd(),
|
||||
env: { ...process.env, ...options.env },
|
||||
shell: true
|
||||
};
|
||||
|
||||
// When using shell: true, pass the full command as a single string
|
||||
const fullCommand = args.length > 0 ? `${command} ${args.join(' ')}` : command;
|
||||
const child = spawn(fullCommand, [], spawnOptions);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
// When using shell: true, pass the full command as a single string
|
||||
const fullCommand =
|
||||
args.length > 0 ? `${command} ${args.join(' ')}` : command;
|
||||
const child = spawn(fullCommand, [], spawnOptions);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (exitCode) => {
|
||||
const output = stdout + stderr;
|
||||
|
||||
// Extract and log costs
|
||||
this.logger.extractAndAddCost(output);
|
||||
|
||||
resolve({ stdout, stderr, exitCode });
|
||||
});
|
||||
child.on('close', (exitCode) => {
|
||||
const output = stdout + stderr;
|
||||
|
||||
// Handle timeout
|
||||
if (options.timeout) {
|
||||
setTimeout(() => {
|
||||
child.kill('SIGTERM');
|
||||
}, options.timeout);
|
||||
}
|
||||
});
|
||||
}
|
||||
// Extract and log costs
|
||||
this.logger.extractAndAddCost(output);
|
||||
|
||||
/**
|
||||
* Execute task-master command
|
||||
* @param {string} subcommand - Task-master subcommand
|
||||
* @param {string[]} args - Command arguments
|
||||
* @param {Object} options - Execution options
|
||||
*/
|
||||
async taskMaster(subcommand, args = [], options = {}) {
|
||||
const fullArgs = [subcommand, ...args];
|
||||
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
|
||||
|
||||
const result = await this.executeCommand('task-master', fullArgs, options);
|
||||
|
||||
if (result.exitCode !== 0 && !options.allowFailure) {
|
||||
this.logger.error(`Command failed with exit code ${result.exitCode}`);
|
||||
this.logger.error(`stderr: ${result.stderr}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
resolve({ stdout, stderr, exitCode });
|
||||
});
|
||||
|
||||
/**
|
||||
* Check if a file exists
|
||||
*/
|
||||
fileExists(filePath) {
|
||||
return existsSync(filePath);
|
||||
}
|
||||
// Handle timeout
|
||||
if (options.timeout) {
|
||||
setTimeout(() => {
|
||||
child.kill('SIGTERM');
|
||||
}, options.timeout);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read JSON file
|
||||
*/
|
||||
readJson(filePath) {
|
||||
try {
|
||||
const content = readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to read JSON file ${filePath}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Execute task-master command
|
||||
* @param {string} subcommand - Task-master subcommand
|
||||
* @param {string[]} args - Command arguments
|
||||
* @param {Object} options - Execution options
|
||||
*/
|
||||
async taskMaster(subcommand, args = [], options = {}) {
|
||||
const fullArgs = [subcommand, ...args];
|
||||
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
|
||||
|
||||
/**
|
||||
* Copy file
|
||||
*/
|
||||
copyFile(source, destination) {
|
||||
try {
|
||||
copyFileSync(source, destination);
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to copy file from ${source} to ${destination}: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const result = await this.executeCommand('task-master', fullArgs, options);
|
||||
|
||||
/**
|
||||
* Wait for a specified duration
|
||||
*/
|
||||
async wait(milliseconds) {
|
||||
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
if (result.exitCode !== 0 && !options.allowFailure) {
|
||||
this.logger.error(`Command failed with exit code ${result.exitCode}`);
|
||||
this.logger.error(`stderr: ${result.stderr}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify task exists in tasks.json
|
||||
*/
|
||||
verifyTaskExists(tasksFile, taskId, tagName = 'master') {
|
||||
const tasks = this.readJson(tasksFile);
|
||||
if (!tasks || !tasks[tagName]) return false;
|
||||
|
||||
return tasks[tagName].tasks.some(task => task.id === taskId);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get task count for a tag
|
||||
*/
|
||||
getTaskCount(tasksFile, tagName = 'master') {
|
||||
const tasks = this.readJson(tasksFile);
|
||||
if (!tasks || !tasks[tagName]) return 0;
|
||||
|
||||
return tasks[tagName].tasks.length;
|
||||
}
|
||||
/**
|
||||
* Check if a file exists
|
||||
*/
|
||||
fileExists(filePath) {
|
||||
return existsSync(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract task ID from command output
|
||||
*/
|
||||
extractTaskId(output) {
|
||||
const patterns = [
|
||||
/✓ Added new task #(\d+(?:\.\d+)?)/,
|
||||
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
|
||||
/Task (\d+(?:\.\d+)?) Created Successfully/
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
const match = output.match(pattern);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Read JSON file
|
||||
*/
|
||||
readJson(filePath) {
|
||||
try {
|
||||
const content = readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to read JSON file ${filePath}: ${error.message}`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run multiple async operations in parallel
|
||||
*/
|
||||
async runParallel(operations) {
|
||||
return Promise.all(operations);
|
||||
}
|
||||
/**
|
||||
* Copy file
|
||||
*/
|
||||
copyFile(source, destination) {
|
||||
try {
|
||||
copyFileSync(source, destination);
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to copy file from ${source} to ${destination}: ${error.message}`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run operations with concurrency limit
|
||||
*/
|
||||
async runWithConcurrency(operations, limit = 3) {
|
||||
const results = [];
|
||||
const executing = [];
|
||||
|
||||
for (const operation of operations) {
|
||||
const promise = operation().then(result => {
|
||||
executing.splice(executing.indexOf(promise), 1);
|
||||
return result;
|
||||
});
|
||||
|
||||
results.push(promise);
|
||||
executing.push(promise);
|
||||
|
||||
if (executing.length >= limit) {
|
||||
await Promise.race(executing);
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.all(results);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Wait for a specified duration
|
||||
*/
|
||||
async wait(milliseconds) {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify task exists in tasks.json
|
||||
*/
|
||||
verifyTaskExists(tasksFile, taskId, tagName = 'master') {
|
||||
const tasks = this.readJson(tasksFile);
|
||||
if (!tasks || !tasks[tagName]) return false;
|
||||
|
||||
return tasks[tagName].tasks.some((task) => task.id === taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get task count for a tag
|
||||
*/
|
||||
getTaskCount(tasksFile, tagName = 'master') {
|
||||
const tasks = this.readJson(tasksFile);
|
||||
if (!tasks || !tasks[tagName]) return 0;
|
||||
|
||||
return tasks[tagName].tasks.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract task ID from command output
|
||||
*/
|
||||
extractTaskId(output) {
|
||||
const patterns = [
|
||||
/✓ Added new task #(\d+(?:\.\d+)?)/,
|
||||
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
|
||||
/Task (\d+(?:\.\d+)?) Created Successfully/
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
const match = output.match(pattern);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run multiple async operations in parallel
|
||||
*/
|
||||
async runParallel(operations) {
|
||||
return Promise.all(operations);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run operations with concurrency limit
|
||||
*/
|
||||
async runWithConcurrency(operations, limit = 3) {
|
||||
const results = [];
|
||||
const executing = [];
|
||||
|
||||
for (const operation of operations) {
|
||||
const promise = operation().then((result) => {
|
||||
executing.splice(executing.indexOf(promise), 1);
|
||||
return result;
|
||||
});
|
||||
|
||||
results.push(promise);
|
||||
executing.push(promise);
|
||||
|
||||
if (executing.length >= limit) {
|
||||
await Promise.race(executing);
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.all(results);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user