ros/claude-task-master
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: run format

Browse Source
This commit is contained in:
Ralph Khreish
2025-07-08 09:40:30 +03:00
parent 395693af24
commit d4208f372a
12 changed files with 2099 additions and 1776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
tests/e2e/config/test-config.js
View File

@@ -10,56 +10,63 @@ const projectRoot = join(__dirname, '../../..');
dotenvConfig({ path: join(projectRoot, '.env') }); dotenvConfig({ path: join(projectRoot, '.env') });
export const testConfig = { export const testConfig = {
// Paths // Paths
paths: { paths: {
projectRoot, projectRoot,
sourceDir: projectRoot, sourceDir: projectRoot,
baseTestDir: join(projectRoot, 'tests/e2e/_runs'), baseTestDir: join(projectRoot, 'tests/e2e/_runs'),
logDir: join(projectRoot, 'tests/e2e/log'), logDir: join(projectRoot, 'tests/e2e/log'),
samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'), samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'),
mainEnvFile: join(projectRoot, '.env'), mainEnvFile: join(projectRoot, '.env'),
supportedModelsFile: join(projectRoot, 'scripts/modules/supported-models.json') supportedModelsFile: join(
}, projectRoot,
'scripts/modules/supported-models.json'
)
},
// Test settings // Test settings
settings: { settings: {
runVerificationTest: true, runVerificationTest: true,
parallelTestGroups: 4, // Number of parallel test groups parallelTestGroups: 4, // Number of parallel test groups
timeout: 600000, // 10 minutes default timeout timeout: 600000, // 10 minutes default timeout
retryAttempts: 2 retryAttempts: 2
}, },
// Provider test configuration // Provider test configuration
providers: [ providers: [
{ name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] }, { name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] },
{ name: 'openai', model: 'gpt-4o', flags: [] }, { name: 'openai', model: 'gpt-4o', flags: [] },
{ name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] }, { name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] },
{ name: 'perplexity', model: 'sonar-pro', flags: [] }, { name: 'perplexity', model: 'sonar-pro', flags: [] },
{ name: 'xai', model: 'grok-3', flags: [] }, { name: 'xai', model: 'grok-3', flags: [] },
{ name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] } { name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] }
], ],
// Test prompts // Test prompts
prompts: { prompts: {
addTask: 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.', addTask:
updateTask: 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.', 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.',
updateFromTask: 'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.', updateTask:
updateSubtask: 'Implementation note: Remember to handle potential API errors and display a user-friendly message.' 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.',
}, updateFromTask:
'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.',
updateSubtask:
'Implementation note: Remember to handle potential API errors and display a user-friendly message.'
},
// LLM Analysis settings // LLM Analysis settings
llmAnalysis: { llmAnalysis: {
enabled: true, enabled: true,
model: 'claude-3-7-sonnet-20250219', model: 'claude-3-7-sonnet-20250219',
provider: 'anthropic', provider: 'anthropic',
maxTokens: 3072 maxTokens: 3072
} }
}; };
// Export test groups for parallel execution // Export test groups for parallel execution
export const testGroups = { export const testGroups = {
setup: ['setup'], setup: ['setup'],
core: ['core'], core: ['core'],
providers: ['providers'], providers: ['providers'],
advanced: ['advanced'] advanced: ['advanced']
}; };

3
tests/e2e/run-e2e-tests.js
View File

@@ -237,7 +237,8 @@ async function runTests(options) {
} }
// Check if we need to run setup (either explicitly requested or needed for other tests) // Check if we need to run setup (either explicitly requested or needed for other tests)
const needsSetup = testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0); const needsSetup =
testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0);
if (needsSetup) { if (needsSetup) {
// Always run setup if we need a test directory // Always run setup if we need a test directory

356
tests/e2e/runners/parallel-runner.js
View File

@@ -7,205 +7,219 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename); const __dirname = dirname(__filename);
export class ParallelTestRunner extends EventEmitter { export class ParallelTestRunner extends EventEmitter {
constructor(logger) { constructor(logger) {
super(); super();
this.logger = logger; this.logger = logger;
this.workers = []; this.workers = [];
this.results = {}; this.results = {};
} }
/** /**
* Run test groups in parallel * Run test groups in parallel
* @param {Object} testGroups - Groups of tests to run * @param {Object} testGroups - Groups of tests to run
* @param {Object} sharedContext - Shared context for all tests * @param {Object} sharedContext - Shared context for all tests
* @returns {Promise<Object>} Combined results from all test groups * @returns {Promise<Object>} Combined results from all test groups
*/ */
async runTestGroups(testGroups, sharedContext) { async runTestGroups(testGroups, sharedContext) {
const groupNames = Object.keys(testGroups); const groupNames = Object.keys(testGroups);
const workerPromises = []; const workerPromises = [];
this.logger.info(`Starting parallel execution of ${groupNames.length} test groups`); this.logger.info(
`Starting parallel execution of ${groupNames.length} test groups`
);
for (const groupName of groupNames) { for (const groupName of groupNames) {
const workerPromise = this.runTestGroup(groupName, testGroups[groupName], sharedContext); const workerPromise = this.runTestGroup(
workerPromises.push(workerPromise); groupName,
} testGroups[groupName],
sharedContext
);
workerPromises.push(workerPromise);
}
// Wait for all workers to complete // Wait for all workers to complete
const results = await Promise.allSettled(workerPromises); const results = await Promise.allSettled(workerPromises);
// Process results // Process results
const combinedResults = { const combinedResults = {
overall: 'passed', overall: 'passed',
groups: {}, groups: {},
summary: { summary: {
totalGroups: groupNames.length, totalGroups: groupNames.length,
passedGroups: 0, passedGroups: 0,
failedGroups: 0, failedGroups: 0,
errors: [] errors: []
} }
}; };
results.forEach((result, index) => { results.forEach((result, index) => {
const groupName = groupNames[index]; const groupName = groupNames[index];
if (result.status === 'fulfilled') { if (result.status === 'fulfilled') {
combinedResults.groups[groupName] = result.value; combinedResults.groups[groupName] = result.value;
if (result.value.status === 'passed') { if (result.value.status === 'passed') {
combinedResults.summary.passedGroups++; combinedResults.summary.passedGroups++;
} else { } else {
combinedResults.summary.failedGroups++; combinedResults.summary.failedGroups++;
combinedResults.overall = 'failed'; combinedResults.overall = 'failed';
} }
} else { } else {
combinedResults.groups[groupName] = { combinedResults.groups[groupName] = {
status: 'failed', status: 'failed',
error: result.reason.message || 'Unknown error' error: result.reason.message || 'Unknown error'
}; };
combinedResults.summary.failedGroups++; combinedResults.summary.failedGroups++;
combinedResults.summary.errors.push({ combinedResults.summary.errors.push({
group: groupName, group: groupName,
error: result.reason.message error: result.reason.message
}); });
combinedResults.overall = 'failed'; combinedResults.overall = 'failed';
} }
}); });
return combinedResults; return combinedResults;
} }
/** /**
* Run a single test group in a worker thread * Run a single test group in a worker thread
*/ */
async runTestGroup(groupName, testModules, sharedContext) { async runTestGroup(groupName, testModules, sharedContext) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const workerPath = join(__dirname, 'test-worker.js'); const workerPath = join(__dirname, 'test-worker.js');
const worker = new Worker(workerPath, { const worker = new Worker(workerPath, {
workerData: { workerData: {
groupName, groupName,
testModules, testModules,
sharedContext, sharedContext,
logDir: this.logger.logDir, logDir: this.logger.logDir,
testRunId: this.logger.testRunId testRunId: this.logger.testRunId
} }
}); });
this.workers.push(worker); this.workers.push(worker);
// Handle messages from worker // Handle messages from worker
worker.on('message', (message) => { worker.on('message', (message) => {
if (message.type === 'log') { if (message.type === 'log') {
const level = message.level.toLowerCase(); const level = message.level.toLowerCase();
if (typeof this.logger[level] === 'function') { if (typeof this.logger[level] === 'function') {
this.logger[level](message.message); this.logger[level](message.message);
} else { } else {
// Fallback to info if the level doesn't exist // Fallback to info if the level doesn't exist
this.logger.info(message.message); this.logger.info(message.message);
} }
} else if (message.type === 'step') { } else if (message.type === 'step') {
this.logger.step(message.message); this.logger.step(message.message);
} else if (message.type === 'cost') { } else if (message.type === 'cost') {
this.logger.addCost(message.cost); this.logger.addCost(message.cost);
} else if (message.type === 'results') { } else if (message.type === 'results') {
this.results[groupName] = message.results; this.results[groupName] = message.results;
} }
}); });
// Handle worker completion // Handle worker completion
worker.on('exit', (code) => { worker.on('exit', (code) => {
this.workers = this.workers.filter(w => w !== worker); this.workers = this.workers.filter((w) => w !== worker);
if (code === 0) { if (code === 0) {
resolve(this.results[groupName] || { status: 'passed', group: groupName }); resolve(
} else { this.results[groupName] || { status: 'passed', group: groupName }
reject(new Error(`Worker for group ${groupName} exited with code ${code}`)); );
} } else {
}); reject(
new Error(`Worker for group ${groupName} exited with code ${code}`)
);
}
});
// Handle worker errors // Handle worker errors
worker.on('error', (error) => { worker.on('error', (error) => {
this.workers = this.workers.filter(w => w !== worker); this.workers = this.workers.filter((w) => w !== worker);
reject(error); reject(error);
}); });
});
}
}); /**
} * Terminate all running workers
*/
async terminate() {
const terminationPromises = this.workers.map((worker) =>
worker
.terminate()
.catch((err) =>
this.logger.warning(`Failed to terminate worker: ${err.message}`)
)
);
/** await Promise.all(terminationPromises);
* Terminate all running workers this.workers = [];
*/ }
async terminate() {
const terminationPromises = this.workers.map(worker =>
worker.terminate().catch(err =>
this.logger.warning(`Failed to terminate worker: ${err.message}`)
)
);
await Promise.all(terminationPromises);
this.workers = [];
}
} }
/** /**
* Sequential test runner for comparison or fallback * Sequential test runner for comparison or fallback
*/ */
export class SequentialTestRunner { export class SequentialTestRunner {
constructor(logger, helpers) { constructor(logger, helpers) {
this.logger = logger; this.logger = logger;
this.helpers = helpers; this.helpers = helpers;
} }
/** /**
* Run tests sequentially * Run tests sequentially
*/ */
async runTests(testModules, context) { async runTests(testModules, context) {
const results = { const results = {
overall: 'passed', overall: 'passed',
tests: {}, tests: {},
summary: { summary: {
totalTests: testModules.length, totalTests: testModules.length,
passedTests: 0, passedTests: 0,
failedTests: 0, failedTests: 0,
errors: [] errors: []
} }
}; };
for (const testModule of testModules) { for (const testModule of testModules) {
try { try {
this.logger.step(`Running ${testModule} tests`); this.logger.step(`Running ${testModule} tests`);
// Dynamic import of test module // Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`); const testPath = join(
const { default: testFn } = await import(testPath); dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test // Run the test
const testResults = await testFn(this.logger, this.helpers, context); const testResults = await testFn(this.logger, this.helpers, context);
results.tests[testModule] = testResults; results.tests[testModule] = testResults;
if (testResults.status === 'passed') { if (testResults.status === 'passed') {
results.summary.passedTests++; results.summary.passedTests++;
} else { } else {
results.summary.failedTests++; results.summary.failedTests++;
results.overall = 'failed'; results.overall = 'failed';
} }
} catch (error) {
this.logger.error(`Failed to run ${testModule}: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
} catch (error) { return results;
this.logger.error(`Failed to run ${testModule}: ${error.message}`); }
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
return results;
}
} }

193
tests/e2e/runners/test-worker.js
View File

@@ -9,124 +9,127 @@ const __dirname = dirname(__filename);
// Worker logger that sends messages to parent // Worker logger that sends messages to parent
class WorkerLogger extends TestLogger { class WorkerLogger extends TestLogger {
constructor(logDir, testRunId, groupName) { constructor(logDir, testRunId, groupName) {
super(logDir, `${testRunId}_${groupName}`); super(logDir, `${testRunId}_${groupName}`);
this.groupName = groupName; this.groupName = groupName;
} }
log(level, message, options = {}) { log(level, message, options = {}) {
super.log(level, message, options); super.log(level, message, options);
// Send log to parent // Send log to parent
parentPort.postMessage({ parentPort.postMessage({
type: 'log', type: 'log',
level: level.toLowerCase(), level: level.toLowerCase(),
message: `[${this.groupName}] ${message}` message: `[${this.groupName}] ${message}`
}); });
} }
step(message) { step(message) {
super.step(message); super.step(message);
parentPort.postMessage({ parentPort.postMessage({
type: 'step', type: 'step',
message: `[${this.groupName}] ${message}` message: `[${this.groupName}] ${message}`
}); });
} }
addCost(cost) { addCost(cost) {
super.addCost(cost); super.addCost(cost);
parentPort.postMessage({ parentPort.postMessage({
type: 'cost', type: 'cost',
cost cost
}); });
} }
} }
// Main worker execution // Main worker execution
async function runTestGroup() { async function runTestGroup() {
const { groupName, testModules, sharedContext, logDir, testRunId } = workerData; const { groupName, testModules, sharedContext, logDir, testRunId } =
workerData;
const logger = new WorkerLogger(logDir, testRunId, groupName); const logger = new WorkerLogger(logDir, testRunId, groupName);
const helpers = new TestHelpers(logger); const helpers = new TestHelpers(logger);
logger.info(`Worker started for test group: ${groupName}`); logger.info(`Worker started for test group: ${groupName}`);
const results = { const results = {
group: groupName, group: groupName,
status: 'passed', status: 'passed',
tests: {}, tests: {},
errors: [], errors: [],
startTime: Date.now() startTime: Date.now()
}; };
try { try {
// Run each test module in the group // Run each test module in the group
for (const testModule of testModules) { for (const testModule of testModules) {
try { try {
logger.info(`Running test: ${testModule}`); logger.info(`Running test: ${testModule}`);
// Dynamic import of test module // Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`); const testPath = join(
const { default: testFn } = await import(testPath); dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test with shared context // Run the test with shared context
const testResults = await testFn(logger, helpers, sharedContext); const testResults = await testFn(logger, helpers, sharedContext);
results.tests[testModule] = testResults; results.tests[testModule] = testResults;
if (testResults.status !== 'passed') { if (testResults.status !== 'passed') {
results.status = 'failed'; results.status = 'failed';
if (testResults.errors) { if (testResults.errors) {
results.errors.push(...testResults.errors); results.errors.push(...testResults.errors);
} }
} }
} catch (error) {
logger.error(`Test ${testModule} failed: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) {
logger.error(`Worker error: ${error.message}`);
results.status = 'failed';
results.errors.push({
group: groupName,
error: error.message,
stack: error.stack
});
}
} catch (error) { results.endTime = Date.now();
logger.error(`Test ${testModule} failed: ${error.message}`); results.duration = results.endTime - results.startTime;
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) { // Flush logs and get summary
logger.error(`Worker error: ${error.message}`); logger.flush();
results.status = 'failed'; const summary = logger.getSummary();
results.errors.push({ results.summary = summary;
group: groupName,
error: error.message,
stack: error.stack
});
}
results.endTime = Date.now(); // Send results to parent
results.duration = results.endTime - results.startTime; parentPort.postMessage({
type: 'results',
results
});
// Flush logs and get summary logger.info(`Worker completed for test group: ${groupName}`);
logger.flush();
const summary = logger.getSummary();
results.summary = summary;
// Send results to parent
parentPort.postMessage({
type: 'results',
results
});
logger.info(`Worker completed for test group: ${groupName}`);
} }
// Run the test group // Run the test group
runTestGroup().catch(error => { runTestGroup().catch((error) => {
console.error('Worker fatal error:', error); console.error('Worker fatal error:', error);
process.exit(1); process.exit(1);
}); });

671
tests/e2e/tests/advanced.test.js
View File

@@ -1,316 +1,443 @@
export default async function testAdvanced(logger, helpers, context) { export default async function testAdvanced(logger, helpers, context) {
const { testDir } = context; const { testDir } = context;
logger.info('Starting advanced features tests...'); logger.info('Starting advanced features tests...');
const results = []; const results = [];
try { try {
// Test Tag Context Management // Test Tag Context Management
logger.info('Testing tag context management...'); logger.info('Testing tag context management...');
// Create new tag contexts // Create new tag contexts
const tag1Result = await helpers.taskMaster('add-tag', ['feature-auth', '--description', 'Authentication feature'], { cwd: testDir }); const tag1Result = await helpers.taskMaster(
results.push({ 'add-tag',
test: 'Create tag context - feature-auth', ['feature-auth', '--description', 'Authentication feature'],
passed: tag1Result.exitCode === 0, { cwd: testDir }
output: tag1Result.stdout );
}); results.push({
test: 'Create tag context - feature-auth',
passed: tag1Result.exitCode === 0,
output: tag1Result.stdout
});
const tag2Result = await helpers.taskMaster('add-tag', ['feature-api', '--description', 'API development'], { cwd: testDir }); const tag2Result = await helpers.taskMaster(
results.push({ 'add-tag',
test: 'Create tag context - feature-api', ['feature-api', '--description', 'API development'],
passed: tag2Result.exitCode === 0, { cwd: testDir }
output: tag2Result.stdout );
}); results.push({
test: 'Create tag context - feature-api',
passed: tag2Result.exitCode === 0,
output: tag2Result.stdout
});
// Add task to feature-auth tag // Add task to feature-auth tag
const task1Result = await helpers.taskMaster('add-task', ['--tag=feature-auth', '--prompt', 'Implement user authentication'], { cwd: testDir }); const task1Result = await helpers.taskMaster(
results.push({ 'add-task',
test: 'Add task to feature-auth tag', ['--tag=feature-auth', '--prompt', 'Implement user authentication'],
passed: task1Result.exitCode === 0, { cwd: testDir }
output: task1Result.stdout );
}); results.push({
test: 'Add task to feature-auth tag',
passed: task1Result.exitCode === 0,
output: task1Result.stdout
});
// Add task to feature-api tag // Add task to feature-api tag
const task2Result = await helpers.taskMaster('add-task', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'], { cwd: testDir }); const task2Result = await helpers.taskMaster(
results.push({ 'add-task',
test: 'Add task to feature-api tag', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'],
passed: task2Result.exitCode === 0, { cwd: testDir }
output: task2Result.stdout );
}); results.push({
test: 'Add task to feature-api tag',
passed: task2Result.exitCode === 0,
output: task2Result.stdout
});
// List all tag contexts // List all tag contexts
const listTagsResult = await helpers.taskMaster('tags', [], { cwd: testDir }); const listTagsResult = await helpers.taskMaster('tags', [], {
results.push({ cwd: testDir
test: 'List all tag contexts', });
passed: listTagsResult.exitCode === 0 && results.push({
listTagsResult.stdout.includes('feature-auth') && test: 'List all tag contexts',
listTagsResult.stdout.includes('feature-api'), passed:
output: listTagsResult.stdout listTagsResult.exitCode === 0 &&
}); listTagsResult.stdout.includes('feature-auth') &&
listTagsResult.stdout.includes('feature-api'),
output: listTagsResult.stdout
});
// List tasks in feature-auth tag // List tasks in feature-auth tag
const taggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-auth'], { cwd: testDir }); const taggedTasksResult = await helpers.taskMaster(
results.push({ 'list',
test: 'List tasks in feature-auth tag', ['--tag=feature-auth'],
passed: taggedTasksResult.exitCode === 0 && { cwd: testDir }
taggedTasksResult.stdout.includes('Implement user authentication'), );
output: taggedTasksResult.stdout results.push({
}); test: 'List tasks in feature-auth tag',
passed:
taggedTasksResult.exitCode === 0 &&
taggedTasksResult.stdout.includes('Implement user authentication'),
output: taggedTasksResult.stdout
});
// Test Model Configuration // Test Model Configuration
logger.info('Testing model configuration...'); logger.info('Testing model configuration...');
// Set main model // Set main model
const setMainModelResult = await helpers.taskMaster('models', ['--set-main', 'gpt-4'], { cwd: testDir }); const setMainModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Set main model', ['--set-main', 'gpt-4'],
passed: setMainModelResult.exitCode === 0, { cwd: testDir }
output: setMainModelResult.stdout );
}); results.push({
test: 'Set main model',
passed: setMainModelResult.exitCode === 0,
output: setMainModelResult.stdout
});
// Set research model // Set research model
const setResearchModelResult = await helpers.taskMaster('models', ['--set-research', 'claude-3-sonnet'], { cwd: testDir }); const setResearchModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Set research model', ['--set-research', 'claude-3-sonnet'],
passed: setResearchModelResult.exitCode === 0, { cwd: testDir }
output: setResearchModelResult.stdout );
}); results.push({
test: 'Set research model',
passed: setResearchModelResult.exitCode === 0,
output: setResearchModelResult.stdout
});
// Set fallback model // Set fallback model
const setFallbackModelResult = await helpers.taskMaster('models', ['--set-fallback', 'gpt-3.5-turbo'], { cwd: testDir }); const setFallbackModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Set fallback model', ['--set-fallback', 'gpt-3.5-turbo'],
passed: setFallbackModelResult.exitCode === 0, { cwd: testDir }
output: setFallbackModelResult.stdout );
}); results.push({
test: 'Set fallback model',
passed: setFallbackModelResult.exitCode === 0,
output: setFallbackModelResult.stdout
});
// Verify model configuration // Verify model configuration
const showModelsResult = await helpers.taskMaster('models', [], { cwd: testDir }); const showModelsResult = await helpers.taskMaster('models', [], {
results.push({ cwd: testDir
test: 'Show model configuration', });
passed: showModelsResult.exitCode === 0 && results.push({
showModelsResult.stdout.includes('gpt-4') && test: 'Show model configuration',
showModelsResult.stdout.includes('claude-3-sonnet') && passed:
showModelsResult.stdout.includes('gpt-3.5-turbo'), showModelsResult.exitCode === 0 &&
output: showModelsResult.stdout showModelsResult.stdout.includes('gpt-4') &&
}); showModelsResult.stdout.includes('claude-3-sonnet') &&
showModelsResult.stdout.includes('gpt-3.5-turbo'),
output: showModelsResult.stdout
});
// Test Task Expansion // Test Task Expansion
logger.info('Testing task expansion...'); logger.info('Testing task expansion...');
// Add task for expansion // Add task for expansion
const expandTaskResult = await helpers.taskMaster('add-task', ['--prompt', 'Build REST API'], { cwd: testDir }); const expandTaskResult = await helpers.taskMaster(
const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/); 'add-task',
const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null; ['--prompt', 'Build REST API'],
{ cwd: testDir }
);
const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/);
const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null;
results.push({ results.push({
test: 'Add task for expansion', test: 'Add task for expansion',
passed: expandTaskResult.exitCode === 0 && expandTaskId !== null, passed: expandTaskResult.exitCode === 0 && expandTaskId !== null,
output: expandTaskResult.stdout output: expandTaskResult.stdout
}); });
if (expandTaskId) { if (expandTaskId) {
// Single task expansion // Single task expansion
const expandResult = await helpers.taskMaster('expand', [expandTaskId], { cwd: testDir }); const expandResult = await helpers.taskMaster('expand', [expandTaskId], {
results.push({ cwd: testDir
test: 'Expand single task', });
passed: expandResult.exitCode === 0 && expandResult.stdout.includes('subtasks'), results.push({
output: expandResult.stdout test: 'Expand single task',
}); passed:
expandResult.exitCode === 0 &&
expandResult.stdout.includes('subtasks'),
output: expandResult.stdout
});
// Verify expand worked // Verify expand worked
const afterExpandResult = await helpers.taskMaster('show', [expandTaskId], { cwd: testDir }); const afterExpandResult = await helpers.taskMaster(
results.push({ 'show',
test: 'Verify task expansion', [expandTaskId],
passed: afterExpandResult.exitCode === 0 && afterExpandResult.stdout.includes('subtasks'), { cwd: testDir }
output: afterExpandResult.stdout );
}); results.push({
test: 'Verify task expansion',
passed:
afterExpandResult.exitCode === 0 &&
afterExpandResult.stdout.includes('subtasks'),
output: afterExpandResult.stdout
});
// Force expand (re-expand) // Force expand (re-expand)
const forceExpandResult = await helpers.taskMaster('expand', [expandTaskId, '--force'], { cwd: testDir }); const forceExpandResult = await helpers.taskMaster(
results.push({ 'expand',
test: 'Force expand task', [expandTaskId, '--force'],
passed: forceExpandResult.exitCode === 0, { cwd: testDir }
output: forceExpandResult.stdout );
}); results.push({
} test: 'Force expand task',
passed: forceExpandResult.exitCode === 0,
output: forceExpandResult.stdout
});
}
// Test Subtask Management // Test Subtask Management
logger.info('Testing subtask management...'); logger.info('Testing subtask management...');
// Add task for subtask testing // Add task for subtask testing
const subtaskParentResult = await helpers.taskMaster('add-task', ['--prompt', 'Create user interface'], { cwd: testDir }); const subtaskParentResult = await helpers.taskMaster(
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/); 'add-task',
const parentTaskId = parentMatch ? parentMatch[1] : null; ['--prompt', 'Create user interface'],
{ cwd: testDir }
);
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/);
const parentTaskId = parentMatch ? parentMatch[1] : null;
if (parentTaskId) { if (parentTaskId) {
// Add subtasks manually // Add subtasks manually
const addSubtask1Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Design mockups'], { cwd: testDir }); const addSubtask1Result = await helpers.taskMaster(
results.push({ 'add-subtask',
test: 'Add subtask - Design mockups', ['--parent', parentTaskId, '--title', 'Design mockups'],
passed: addSubtask1Result.exitCode === 0, { cwd: testDir }
output: addSubtask1Result.stdout );
}); results.push({
test: 'Add subtask - Design mockups',
passed: addSubtask1Result.exitCode === 0,
output: addSubtask1Result.stdout
});
const addSubtask2Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Implement components'], { cwd: testDir }); const addSubtask2Result = await helpers.taskMaster(
results.push({ 'add-subtask',
test: 'Add subtask - Implement components', ['--parent', parentTaskId, '--title', 'Implement components'],
passed: addSubtask2Result.exitCode === 0, { cwd: testDir }
output: addSubtask2Result.stdout );
}); results.push({
test: 'Add subtask - Implement components',
passed: addSubtask2Result.exitCode === 0,
output: addSubtask2Result.stdout
});
// List subtasks (use show command to see subtasks) // List subtasks (use show command to see subtasks)
const listSubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); const listSubtasksResult = await helpers.taskMaster(
results.push({ 'show',
test: 'List subtasks', [parentTaskId],
passed: listSubtasksResult.exitCode === 0 && { cwd: testDir }
listSubtasksResult.stdout.includes('Design mockups') && );
listSubtasksResult.stdout.includes('Implement components'), results.push({
output: listSubtasksResult.stdout test: 'List subtasks',
}); passed:
listSubtasksResult.exitCode === 0 &&
listSubtasksResult.stdout.includes('Design mockups') &&
listSubtasksResult.stdout.includes('Implement components'),
output: listSubtasksResult.stdout
});
// Update subtask // Update subtask
const subtaskId = `${parentTaskId}.1`; const subtaskId = `${parentTaskId}.1`;
const updateSubtaskResult = await helpers.taskMaster('update-subtask', ['--id', subtaskId, '--prompt', 'Create detailed mockups'], { cwd: testDir }); const updateSubtaskResult = await helpers.taskMaster(
results.push({ 'update-subtask',
test: 'Update subtask', ['--id', subtaskId, '--prompt', 'Create detailed mockups'],
passed: updateSubtaskResult.exitCode === 0, { cwd: testDir }
output: updateSubtaskResult.stdout );
}); results.push({
test: 'Update subtask',
passed: updateSubtaskResult.exitCode === 0,
output: updateSubtaskResult.stdout
});
// Remove subtask // Remove subtask
const removeSubtaskId = `${parentTaskId}.2`; const removeSubtaskId = `${parentTaskId}.2`;
const removeSubtaskResult = await helpers.taskMaster('remove-subtask', ['--id', removeSubtaskId], { cwd: testDir }); const removeSubtaskResult = await helpers.taskMaster(
results.push({ 'remove-subtask',
test: 'Remove subtask', ['--id', removeSubtaskId],
passed: removeSubtaskResult.exitCode === 0, { cwd: testDir }
output: removeSubtaskResult.stdout );
}); results.push({
test: 'Remove subtask',
passed: removeSubtaskResult.exitCode === 0,
output: removeSubtaskResult.stdout
});
// Verify subtask changes // Verify subtask changes
const verifySubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); const verifySubtasksResult = await helpers.taskMaster(
results.push({ 'show',
test: 'Verify subtask changes', [parentTaskId],
passed: verifySubtasksResult.exitCode === 0 && { cwd: testDir }
verifySubtasksResult.stdout.includes('Create detailed mockups') && );
!verifySubtasksResult.stdout.includes('Implement components'), results.push({
output: verifySubtasksResult.stdout test: 'Verify subtask changes',
}); passed:
verifySubtasksResult.exitCode === 0 &&
verifySubtasksResult.stdout.includes('Create detailed mockups') &&
!verifySubtasksResult.stdout.includes('Implement components'),
output: verifySubtasksResult.stdout
});
// Clear all subtasks // Clear all subtasks
const clearSubtasksResult = await helpers.taskMaster('clear-subtasks', ['--id', parentTaskId], { cwd: testDir }); const clearSubtasksResult = await helpers.taskMaster(
results.push({ 'clear-subtasks',
test: 'Clear all subtasks', ['--id', parentTaskId],
passed: clearSubtasksResult.exitCode === 0, { cwd: testDir }
output: clearSubtasksResult.stdout );
}); results.push({
test: 'Clear all subtasks',
passed: clearSubtasksResult.exitCode === 0,
output: clearSubtasksResult.stdout
});
// Verify subtasks cleared // Verify subtasks cleared
const verifyClearResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); const verifyClearResult = await helpers.taskMaster(
results.push({ 'show',
test: 'Verify subtasks cleared', [parentTaskId],
passed: verifyClearResult.exitCode === 0 && { cwd: testDir }
(!verifyClearResult.stdout.includes('Design mockups') && );
!verifyClearResult.stdout.includes('Create detailed mockups')), results.push({
output: verifyClearResult.stdout test: 'Verify subtasks cleared',
}); passed:
} verifyClearResult.exitCode === 0 &&
!verifyClearResult.stdout.includes('Design mockups') &&
!verifyClearResult.stdout.includes('Create detailed mockups'),
output: verifyClearResult.stdout
});
}
// Test Expand All // Test Expand All
logger.info('Testing expand all...'); logger.info('Testing expand all...');
// Add multiple tasks // Add multiple tasks
await helpers.taskMaster('add-task', ['--prompt', 'Task A for expand all'], { cwd: testDir }); await helpers.taskMaster(
await helpers.taskMaster('add-task', ['--prompt', 'Task B for expand all'], { cwd: testDir }); 'add-task',
['--prompt', 'Task A for expand all'],
{ cwd: testDir }
);
await helpers.taskMaster(
'add-task',
['--prompt', 'Task B for expand all'],
{ cwd: testDir }
);
const expandAllResult = await helpers.taskMaster('expand', ['--all'], { cwd: testDir }); const expandAllResult = await helpers.taskMaster('expand', ['--all'], {
results.push({ cwd: testDir
test: 'Expand all tasks', });
passed: expandAllResult.exitCode === 0, results.push({
output: expandAllResult.stdout test: 'Expand all tasks',
}); passed: expandAllResult.exitCode === 0,
output: expandAllResult.stdout
});
// Test Generate Task Files // Test Generate Task Files
logger.info('Testing generate task files...'); logger.info('Testing generate task files...');
// Generate files for a specific task // Generate files for a specific task
if (expandTaskId) { if (expandTaskId) {
const generateResult = await helpers.taskMaster('generate', [expandTaskId], { cwd: testDir }); const generateResult = await helpers.taskMaster(
results.push({ 'generate',
test: 'Generate task files', [expandTaskId],
passed: generateResult.exitCode === 0, { cwd: testDir }
output: generateResult.stdout );
}); results.push({
test: 'Generate task files',
passed: generateResult.exitCode === 0,
output: generateResult.stdout
});
// Check if files were created // Check if files were created
const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`; const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`;
const fileExists = helpers.fileExists(taskFilePath); const fileExists = helpers.fileExists(taskFilePath);
results.push({ results.push({
test: 'Verify generated task file exists', test: 'Verify generated task file exists',
passed: fileExists, passed: fileExists,
output: fileExists ? `Task file created at ${taskFilePath}` : 'Task file not found' output: fileExists
}); ? `Task file created at ${taskFilePath}`
} : 'Task file not found'
});
}
// Test Tag Context Integrity After Operations // Test Tag Context Integrity After Operations
logger.info('Testing tag context integrity after operations...'); logger.info('Testing tag context integrity after operations...');
// Verify tag contexts still exist // Verify tag contexts still exist
const finalTagListResult = await helpers.taskMaster('tags', [], { cwd: testDir }); const finalTagListResult = await helpers.taskMaster('tags', [], {
results.push({ cwd: testDir
test: 'Final tag context list verification', });
passed: finalTagListResult.exitCode === 0 && results.push({
finalTagListResult.stdout.includes('feature-auth') && test: 'Final tag context list verification',
finalTagListResult.stdout.includes('feature-api'), passed:
output: finalTagListResult.stdout finalTagListResult.exitCode === 0 &&
}); finalTagListResult.stdout.includes('feature-auth') &&
finalTagListResult.stdout.includes('feature-api'),
output: finalTagListResult.stdout
});
// Verify tasks are still in their respective tag contexts // Verify tasks are still in their respective tag contexts
const finalTaggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-api'], { cwd: testDir }); const finalTaggedTasksResult = await helpers.taskMaster(
results.push({ 'list',
test: 'Final tasks in tag context verification', ['--tag=feature-api'],
passed: finalTaggedTasksResult.exitCode === 0 && { cwd: testDir }
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'), );
output: finalTaggedTasksResult.stdout results.push({
}); test: 'Final tasks in tag context verification',
passed:
finalTaggedTasksResult.exitCode === 0 &&
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'),
output: finalTaggedTasksResult.stdout
});
// Test Additional Advanced Features // Test Additional Advanced Features
logger.info('Testing additional advanced features...'); logger.info('Testing additional advanced features...');
// Test priority task // Test priority task
const priorityTagResult = await helpers.taskMaster('add-task', ['--prompt', 'High priority task', '--priority', 'high'], { cwd: testDir }); const priorityTagResult = await helpers.taskMaster(
results.push({ 'add-task',
test: 'Add task with high priority', ['--prompt', 'High priority task', '--priority', 'high'],
passed: priorityTagResult.exitCode === 0, { cwd: testDir }
output: priorityTagResult.stdout );
}); results.push({
test: 'Add task with high priority',
passed: priorityTagResult.exitCode === 0,
output: priorityTagResult.stdout
});
// Test filtering by status // Test filtering by status
const statusFilterResult = await helpers.taskMaster('list', ['--status', 'pending'], { cwd: testDir }); const statusFilterResult = await helpers.taskMaster(
results.push({ 'list',
test: 'Filter by status', ['--status', 'pending'],
passed: statusFilterResult.exitCode === 0, { cwd: testDir }
output: statusFilterResult.stdout );
}); results.push({
test: 'Filter by status',
passed: statusFilterResult.exitCode === 0,
output: statusFilterResult.stdout
});
} catch (error) {
logger.error('Error in advanced features tests:', error);
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
} catch (error) { const passed = results.filter((r) => r.passed).length;
logger.error('Error in advanced features tests:', error); const total = results.length;
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
const passed = results.filter(r => r.passed).length; return {
const total = results.length; name: 'Advanced Features',
passed,
return { total,
name: 'Advanced Features', results,
passed, summary: `Advanced features tests: ${passed}/${total} passed`
total, };
results, }
summary: `Advanced features tests: ${passed}/${total} passed`
};
};

592
tests/e2e/tests/core.test.js
View File

@@ -4,282 +4,376 @@
*/ */
export default async function testCoreOperations(logger, helpers, context) { export default async function testCoreOperations(logger, helpers, context) {
const { testDir } = context; const { testDir } = context;
const results = { const results = {
status: 'passed', status: 'passed',
errors: [] errors: []
}; };
try { try {
logger.info('Starting core task operations tests...'); logger.info('Starting core task operations tests...');
// Test 1: List tasks (may have tasks from PRD parsing) // Test 1: List tasks (may have tasks from PRD parsing)
logger.info('\nTest 1: List tasks'); logger.info('\nTest 1: List tasks');
const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir }); const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult1.exitCode !== 0) { if (listResult1.exitCode !== 0) {
throw new Error(`List command failed: ${listResult1.stderr}`); throw new Error(`List command failed: ${listResult1.stderr}`);
} }
// Check for expected output patterns - either empty or with tasks // Check for expected output patterns - either empty or with tasks
const hasValidOutput = listResult1.stdout.includes('No tasks found') || const hasValidOutput =
listResult1.stdout.includes('Task List') || listResult1.stdout.includes('No tasks found') ||
listResult1.stdout.includes('Project Dashboard') || listResult1.stdout.includes('Task List') ||
listResult1.stdout.includes('Listing tasks from'); listResult1.stdout.includes('Project Dashboard') ||
if (!hasValidOutput) { listResult1.stdout.includes('Listing tasks from');
throw new Error('Unexpected list output format'); if (!hasValidOutput) {
} throw new Error('Unexpected list output format');
logger.success('✓ List tasks successful'); }
logger.success('✓ List tasks successful');
// Test 2: Add manual task // Test 2: Add manual task
logger.info('\nTest 2: Add manual task'); logger.info('\nTest 2: Add manual task');
const addResult1 = await helpers.taskMaster('add-task', ['--title', 'Write unit tests', '--description', 'Create comprehensive unit tests for the application'], { cwd: testDir }); const addResult1 = await helpers.taskMaster(
if (addResult1.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add manual task: ${addResult1.stderr}`); [
} '--title',
const manualTaskId = helpers.extractTaskId(addResult1.stdout); 'Write unit tests',
if (!manualTaskId) { '--description',
throw new Error('Failed to extract task ID from add output'); 'Create comprehensive unit tests for the application'
} ],
logger.success(`✓ Added manual task with ID: ${manualTaskId}`); { cwd: testDir }
);
if (addResult1.exitCode !== 0) {
throw new Error(`Failed to add manual task: ${addResult1.stderr}`);
}
const manualTaskId = helpers.extractTaskId(addResult1.stdout);
if (!manualTaskId) {
throw new Error('Failed to extract task ID from add output');
}
logger.success(`✓ Added manual task with ID: ${manualTaskId}`);
// Test 3: Add AI task // Test 3: Add AI task
logger.info('\nTest 3: Add AI task'); logger.info('\nTest 3: Add AI task');
const addResult2 = await helpers.taskMaster('add-task', ['--prompt', 'Implement authentication system'], { cwd: testDir }); const addResult2 = await helpers.taskMaster(
if (addResult2.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add AI task: ${addResult2.stderr}`); ['--prompt', 'Implement authentication system'],
} { cwd: testDir }
const aiTaskId = helpers.extractTaskId(addResult2.stdout); );
if (!aiTaskId) { if (addResult2.exitCode !== 0) {
throw new Error('Failed to extract AI task ID from add output'); throw new Error(`Failed to add AI task: ${addResult2.stderr}`);
} }
logger.success(`✓ Added AI task with ID: ${aiTaskId}`); const aiTaskId = helpers.extractTaskId(addResult2.stdout);
if (!aiTaskId) {
throw new Error('Failed to extract AI task ID from add output');
}
logger.success(`✓ Added AI task with ID: ${aiTaskId}`);
// Test 4: Add another task for dependency testing // Test 4: Add another task for dependency testing
logger.info('\nTest 4: Add task for dependency testing'); logger.info('\nTest 4: Add task for dependency testing');
const addResult3 = await helpers.taskMaster('add-task', ['--title', 'Create database schema', '--description', 'Design and implement the database schema'], { cwd: testDir }); const addResult3 = await helpers.taskMaster(
if (addResult3.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add database task: ${addResult3.stderr}`); [
} '--title',
const dbTaskId = helpers.extractTaskId(addResult3.stdout); 'Create database schema',
if (!dbTaskId) { '--description',
throw new Error('Failed to extract database task ID'); 'Design and implement the database schema'
} ],
logger.success(`✓ Added database task with ID: ${dbTaskId}`); { cwd: testDir }
);
if (addResult3.exitCode !== 0) {
throw new Error(`Failed to add database task: ${addResult3.stderr}`);
}
const dbTaskId = helpers.extractTaskId(addResult3.stdout);
if (!dbTaskId) {
throw new Error('Failed to extract database task ID');
}
logger.success(`✓ Added database task with ID: ${dbTaskId}`);
// Test 5: List tasks (should show our newly added tasks) // Test 5: List tasks (should show our newly added tasks)
logger.info('\nTest 5: List all tasks'); logger.info('\nTest 5: List all tasks');
const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir }); const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult2.exitCode !== 0) { if (listResult2.exitCode !== 0) {
throw new Error(`List command failed: ${listResult2.stderr}`); throw new Error(`List command failed: ${listResult2.stderr}`);
} }
// Check that we can find our task IDs in the output // Check that we can find our task IDs in the output
const hasTask11 = listResult2.stdout.includes('11'); const hasTask11 = listResult2.stdout.includes('11');
const hasTask12 = listResult2.stdout.includes('12'); const hasTask12 = listResult2.stdout.includes('12');
const hasTask13 = listResult2.stdout.includes('13'); const hasTask13 = listResult2.stdout.includes('13');
if (!hasTask11 || !hasTask12 || !hasTask13) { if (!hasTask11 || !hasTask12 || !hasTask13) {
throw new Error('Not all task IDs found in list output'); throw new Error('Not all task IDs found in list output');
} }
// Also check for partial matches (list may truncate titles) // Also check for partial matches (list may truncate titles)
const hasOurTasks = listResult2.stdout.includes('Write') || const hasOurTasks =
listResult2.stdout.includes('Create'); listResult2.stdout.includes('Write') ||
if (hasOurTasks) { listResult2.stdout.includes('Create');
logger.success('✓ List tasks shows our added tasks'); if (hasOurTasks) {
} else { logger.success('✓ List tasks shows our added tasks');
logger.warning('Task titles may be truncated in list view'); } else {
} logger.warning('Task titles may be truncated in list view');
}
// Test 6: Get next task // Test 6: Get next task
logger.info('\nTest 6: Get next task'); logger.info('\nTest 6: Get next task');
const nextResult = await helpers.taskMaster('next', [], { cwd: testDir }); const nextResult = await helpers.taskMaster('next', [], { cwd: testDir });
if (nextResult.exitCode !== 0) { if (nextResult.exitCode !== 0) {
throw new Error(`Next task command failed: ${nextResult.stderr}`); throw new Error(`Next task command failed: ${nextResult.stderr}`);
} }
logger.success('✓ Get next task successful'); logger.success('✓ Get next task successful');
// Test 7: Show task details // Test 7: Show task details
logger.info('\nTest 7: Show task details'); logger.info('\nTest 7: Show task details');
const showResult = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); const showResult = await helpers.taskMaster('show', [aiTaskId], {
if (showResult.exitCode !== 0) { cwd: testDir
throw new Error(`Show task details failed: ${showResult.stderr}`); });
} if (showResult.exitCode !== 0) {
// Check that the task ID is shown and basic structure is present throw new Error(`Show task details failed: ${showResult.stderr}`);
if (!showResult.stdout.includes(`Task: #${aiTaskId}`) && !showResult.stdout.includes(`ID: │ ${aiTaskId}`)) { }
throw new Error('Task ID not found in show output'); // Check that the task ID is shown and basic structure is present
} if (
if (!showResult.stdout.includes('Status:') || !showResult.stdout.includes('Priority:')) { !showResult.stdout.includes(`Task: #${aiTaskId}`) &&
throw new Error('Task details missing expected fields'); !showResult.stdout.includes(`ID: │ ${aiTaskId}`)
} ) {
logger.success('✓ Show task details successful'); throw new Error('Task ID not found in show output');
}
if (
!showResult.stdout.includes('Status:') ||
!showResult.stdout.includes('Priority:')
) {
throw new Error('Task details missing expected fields');
}
logger.success('✓ Show task details successful');
// Test 8: Add dependencies // Test 8: Add dependencies
logger.info('\nTest 8: Add dependencies'); logger.info('\nTest 8: Add dependencies');
const addDepResult = await helpers.taskMaster('add-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); const addDepResult = await helpers.taskMaster(
if (addDepResult.exitCode !== 0) { 'add-dependency',
throw new Error(`Failed to add dependency: ${addDepResult.stderr}`); ['--id', aiTaskId, '--depends-on', dbTaskId],
} { cwd: testDir }
logger.success('✓ Added dependency successfully'); );
if (addDepResult.exitCode !== 0) {
throw new Error(`Failed to add dependency: ${addDepResult.stderr}`);
}
logger.success('✓ Added dependency successfully');
// Test 9: Verify dependency was added // Test 9: Verify dependency was added
logger.info('\nTest 9: Verify dependency'); logger.info('\nTest 9: Verify dependency');
const showResult2 = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); const showResult2 = await helpers.taskMaster('show', [aiTaskId], {
if (showResult2.exitCode !== 0) { cwd: testDir
throw new Error(`Show task failed: ${showResult2.stderr}`); });
} if (showResult2.exitCode !== 0) {
if (!showResult2.stdout.includes('Dependencies:') || !showResult2.stdout.includes(dbTaskId)) { throw new Error(`Show task failed: ${showResult2.stderr}`);
throw new Error('Dependency not shown in task details'); }
} if (
logger.success('✓ Dependency verified in task details'); !showResult2.stdout.includes('Dependencies:') ||
!showResult2.stdout.includes(dbTaskId)
) {
throw new Error('Dependency not shown in task details');
}
logger.success('✓ Dependency verified in task details');
// Test 10: Test circular dependency (should fail) // Test 10: Test circular dependency (should fail)
logger.info('\nTest 10: Test circular dependency prevention'); logger.info('\nTest 10: Test circular dependency prevention');
const circularResult = await helpers.taskMaster('add-dependency', ['--id', dbTaskId, '--depends-on', aiTaskId], { const circularResult = await helpers.taskMaster(
cwd: testDir, 'add-dependency',
allowFailure: true ['--id', dbTaskId, '--depends-on', aiTaskId],
}); {
if (circularResult.exitCode === 0) { cwd: testDir,
throw new Error('Circular dependency was not prevented'); allowFailure: true
} }
if (!circularResult.stderr.toLowerCase().includes('circular')) { );
throw new Error('Expected circular dependency error message'); if (circularResult.exitCode === 0) {
} throw new Error('Circular dependency was not prevented');
logger.success('✓ Circular dependency prevented successfully'); }
if (!circularResult.stderr.toLowerCase().includes('circular')) {
throw new Error('Expected circular dependency error message');
}
logger.success('✓ Circular dependency prevented successfully');
// Test 11: Test non-existent dependency // Test 11: Test non-existent dependency
logger.info('\nTest 11: Test non-existent dependency'); logger.info('\nTest 11: Test non-existent dependency');
const nonExistResult = await helpers.taskMaster('add-dependency', ['--id', '99999', '--depends-on', '88888'], { const nonExistResult = await helpers.taskMaster(
cwd: testDir, 'add-dependency',
allowFailure: true ['--id', '99999', '--depends-on', '88888'],
}); {
if (nonExistResult.exitCode === 0) { cwd: testDir,
throw new Error('Non-existent dependency was incorrectly allowed'); allowFailure: true
} }
logger.success('✓ Non-existent dependency handled correctly'); );
if (nonExistResult.exitCode === 0) {
throw new Error('Non-existent dependency was incorrectly allowed');
}
logger.success('✓ Non-existent dependency handled correctly');
// Test 12: Remove dependency // Test 12: Remove dependency
logger.info('\nTest 12: Remove dependency'); logger.info('\nTest 12: Remove dependency');
const removeDepResult = await helpers.taskMaster('remove-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); const removeDepResult = await helpers.taskMaster(
if (removeDepResult.exitCode !== 0) { 'remove-dependency',
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`); ['--id', aiTaskId, '--depends-on', dbTaskId],
} { cwd: testDir }
logger.success('✓ Removed dependency successfully'); );
if (removeDepResult.exitCode !== 0) {
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`);
}
logger.success('✓ Removed dependency successfully');
// Test 13: Validate dependencies // Test 13: Validate dependencies
logger.info('\nTest 13: Validate dependencies'); logger.info('\nTest 13: Validate dependencies');
const validateResult = await helpers.taskMaster('validate-dependencies', [], { cwd: testDir }); const validateResult = await helpers.taskMaster(
if (validateResult.exitCode !== 0) { 'validate-dependencies',
throw new Error(`Dependency validation failed: ${validateResult.stderr}`); [],
} { cwd: testDir }
logger.success('✓ Dependency validation successful'); );
if (validateResult.exitCode !== 0) {
throw new Error(`Dependency validation failed: ${validateResult.stderr}`);
}
logger.success('✓ Dependency validation successful');
// Test 14: Update task description // Test 14: Update task description
logger.info('\nTest 14: Update task description'); logger.info('\nTest 14: Update task description');
const updateResult = await helpers.taskMaster('update-task', [manualTaskId, '--description', 'Write comprehensive unit tests'], { cwd: testDir }); const updateResult = await helpers.taskMaster(
if (updateResult.exitCode !== 0) { 'update-task',
throw new Error(`Failed to update task: ${updateResult.stderr}`); [manualTaskId, '--description', 'Write comprehensive unit tests'],
} { cwd: testDir }
logger.success('✓ Updated task description successfully'); );
if (updateResult.exitCode !== 0) {
throw new Error(`Failed to update task: ${updateResult.stderr}`);
}
logger.success('✓ Updated task description successfully');
// Test 15: Add subtask // Test 15: Add subtask
logger.info('\nTest 15: Add subtask'); logger.info('\nTest 15: Add subtask');
const subtaskResult = await helpers.taskMaster('add-subtask', [manualTaskId, 'Write test for login'], { cwd: testDir }); const subtaskResult = await helpers.taskMaster(
if (subtaskResult.exitCode !== 0) { 'add-subtask',
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`); [manualTaskId, 'Write test for login'],
} { cwd: testDir }
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1'; );
logger.success(`✓ Added subtask with ID: ${subtaskId}`); if (subtaskResult.exitCode !== 0) {
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`);
}
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1';
logger.success(`✓ Added subtask with ID: ${subtaskId}`);
// Test 16: Verify subtask relationship // Test 16: Verify subtask relationship
logger.info('\nTest 16: Verify subtask relationship'); logger.info('\nTest 16: Verify subtask relationship');
const showResult3 = await helpers.taskMaster('show', [manualTaskId], { cwd: testDir }); const showResult3 = await helpers.taskMaster('show', [manualTaskId], {
if (showResult3.exitCode !== 0) { cwd: testDir
throw new Error(`Show task failed: ${showResult3.stderr}`); });
} if (showResult3.exitCode !== 0) {
if (!showResult3.stdout.includes('Subtasks:')) { throw new Error(`Show task failed: ${showResult3.stderr}`);
throw new Error('Subtasks section not shown in parent task'); }
} if (!showResult3.stdout.includes('Subtasks:')) {
logger.success('✓ Subtask relationship verified'); throw new Error('Subtasks section not shown in parent task');
}
logger.success('✓ Subtask relationship verified');
// Test 17: Set task status to in_progress // Test 17: Set task status to in_progress
logger.info('\nTest 17: Set task status to in_progress'); logger.info('\nTest 17: Set task status to in_progress');
const statusResult1 = await helpers.taskMaster('set-status', [manualTaskId, 'in_progress'], { cwd: testDir }); const statusResult1 = await helpers.taskMaster(
if (statusResult1.exitCode !== 0) { 'set-status',
throw new Error(`Failed to update task status: ${statusResult1.stderr}`); [manualTaskId, 'in_progress'],
} { cwd: testDir }
logger.success('✓ Set task status to in_progress'); );
if (statusResult1.exitCode !== 0) {
throw new Error(`Failed to update task status: ${statusResult1.stderr}`);
}
logger.success('✓ Set task status to in_progress');
// Test 18: Set task status to completed // Test 18: Set task status to completed
logger.info('\nTest 18: Set task status to completed'); logger.info('\nTest 18: Set task status to completed');
const statusResult2 = await helpers.taskMaster('set-status', [dbTaskId, 'completed'], { cwd: testDir }); const statusResult2 = await helpers.taskMaster(
if (statusResult2.exitCode !== 0) { 'set-status',
throw new Error(`Failed to complete task: ${statusResult2.stderr}`); [dbTaskId, 'completed'],
} { cwd: testDir }
logger.success('✓ Set task status to completed'); );
if (statusResult2.exitCode !== 0) {
throw new Error(`Failed to complete task: ${statusResult2.stderr}`);
}
logger.success('✓ Set task status to completed');
// Test 19: List tasks with status filter // Test 19: List tasks with status filter
logger.info('\nTest 19: List tasks by status'); logger.info('\nTest 19: List tasks by status');
const listStatusResult = await helpers.taskMaster('list', ['--status', 'completed'], { cwd: testDir }); const listStatusResult = await helpers.taskMaster(
if (listStatusResult.exitCode !== 0) { 'list',
throw new Error(`List by status failed: ${listStatusResult.stderr}`); ['--status', 'completed'],
} { cwd: testDir }
if (!listStatusResult.stdout.includes('Create database schema')) { );
throw new Error('Completed task not shown in filtered list'); if (listStatusResult.exitCode !== 0) {
} throw new Error(`List by status failed: ${listStatusResult.stderr}`);
logger.success('✓ List tasks by status successful'); }
if (!listStatusResult.stdout.includes('Create database schema')) {
throw new Error('Completed task not shown in filtered list');
}
logger.success('✓ List tasks by status successful');
// Test 20: Remove single task // Test 20: Remove single task
logger.info('\nTest 20: Remove single task'); logger.info('\nTest 20: Remove single task');
const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], { cwd: testDir }); const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], {
if (removeResult1.exitCode !== 0) { cwd: testDir
throw new Error(`Failed to remove task: ${removeResult1.stderr}`); });
} if (removeResult1.exitCode !== 0) {
logger.success('✓ Removed single task successfully'); throw new Error(`Failed to remove task: ${removeResult1.stderr}`);
}
logger.success('✓ Removed single task successfully');
// Test 21: Remove multiple tasks // Test 21: Remove multiple tasks
logger.info('\nTest 21: Remove multiple tasks'); logger.info('\nTest 21: Remove multiple tasks');
const removeResult2 = await helpers.taskMaster('remove-task', [manualTaskId, aiTaskId], { cwd: testDir }); const removeResult2 = await helpers.taskMaster(
if (removeResult2.exitCode !== 0) { 'remove-task',
throw new Error(`Failed to remove multiple tasks: ${removeResult2.stderr}`); [manualTaskId, aiTaskId],
} { cwd: testDir }
logger.success('✓ Removed multiple tasks successfully'); );
if (removeResult2.exitCode !== 0) {
throw new Error(
`Failed to remove multiple tasks: ${removeResult2.stderr}`
);
}
logger.success('✓ Removed multiple tasks successfully');
// Test 22: Verify tasks were removed // Test 22: Verify tasks were removed
logger.info('\nTest 22: Verify tasks were removed'); logger.info('\nTest 22: Verify tasks were removed');
const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir }); const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult3.exitCode !== 0) { if (listResult3.exitCode !== 0) {
throw new Error(`List command failed: ${listResult3.stderr}`); throw new Error(`List command failed: ${listResult3.stderr}`);
} }
// Check that our specific task IDs are no longer in the list // Check that our specific task IDs are no longer in the list
const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(listResult3.stdout); const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(listResult3.stdout); listResult3.stdout
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(listResult3.stdout); );
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(
listResult3.stdout
);
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(
listResult3.stdout
);
if (stillHasTask11 || stillHasTask12 || stillHasTask13) { if (stillHasTask11 || stillHasTask12 || stillHasTask13) {
throw new Error('Removed task IDs still appear in list'); throw new Error('Removed task IDs still appear in list');
} }
logger.success('✓ Verified tasks were removed'); logger.success('✓ Verified tasks were removed');
// Test 23: Fix dependencies (cleanup) // Test 23: Fix dependencies (cleanup)
logger.info('\nTest 23: Fix dependencies'); logger.info('\nTest 23: Fix dependencies');
const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], { cwd: testDir }); const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], {
if (fixDepsResult.exitCode !== 0) { cwd: testDir
// Non-critical, just log });
logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`); if (fixDepsResult.exitCode !== 0) {
} else { // Non-critical, just log
logger.success('✓ Fix dependencies command executed'); logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`);
} } else {
logger.success('✓ Fix dependencies command executed');
}
logger.info('\n✅ All core task operations tests passed!'); logger.info('\n✅ All core task operations tests passed!');
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'core operations',
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
} catch (error) { return results;
results.status = 'failed';
results.errors.push({
test: 'core operations',
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
return results;
} }

288
tests/e2e/tests/providers.test.js
View File

@@ -4,161 +4,183 @@
*/ */
export default async function testProviders(logger, helpers, context) { export default async function testProviders(logger, helpers, context) {
const { testDir, config } = context; const { testDir, config } = context;
const results = { const results = {
status: 'passed', status: 'passed',
errors: [], errors: [],
providerComparison: {}, providerComparison: {},
summary: { summary: {
totalProviders: 0, totalProviders: 0,
successfulProviders: 0, successfulProviders: 0,
failedProviders: 0, failedProviders: 0,
averageExecutionTime: 0, averageExecutionTime: 0,
successRate: '0%' successRate: '0%'
} }
}; };
try { try {
logger.info('Starting multi-provider tests...'); logger.info('Starting multi-provider tests...');
const providers = config.providers; const providers = config.providers;
const standardPrompt = config.prompts.addTask; const standardPrompt = config.prompts.addTask;
results.summary.totalProviders = providers.length; results.summary.totalProviders = providers.length;
let totalExecutionTime = 0; let totalExecutionTime = 0;
// Process providers in batches to avoid rate limits // Process providers in batches to avoid rate limits
const batchSize = 3; const batchSize = 3;
for (let i = 0; i < providers.length; i += batchSize) { for (let i = 0; i < providers.length; i += batchSize) {
const batch = providers.slice(i, i + batchSize); const batch = providers.slice(i, i + batchSize);
const batchPromises = batch.map(async (provider) => { const batchPromises = batch.map(async (provider) => {
const providerResult = { const providerResult = {
status: 'failed', status: 'failed',
taskId: null, taskId: null,
executionTime: 0, executionTime: 0,
subtaskCount: 0, subtaskCount: 0,
features: { features: {
hasTitle: false, hasTitle: false,
hasDescription: false, hasDescription: false,
hasSubtasks: false, hasSubtasks: false,
hasDependencies: false hasDependencies: false
}, },
error: null, error: null,
taskDetails: null taskDetails: null
}; };
const startTime = Date.now(); const startTime = Date.now();
try { try {
logger.info(`\nTesting provider: ${provider.name} with model: ${provider.model}`); logger.info(
`\nTesting provider: ${provider.name} with model: ${provider.model}`
);
// Step 1: Set the main model for this provider // Step 1: Set the main model for this provider
logger.info(`Setting model to ${provider.model}...`); logger.info(`Setting model to ${provider.model}...`);
const setModelResult = await helpers.taskMaster('models', ['--set-main', provider.model], { cwd: testDir }); const setModelResult = await helpers.taskMaster(
if (setModelResult.exitCode !== 0) { 'models',
throw new Error(`Failed to set model for ${provider.name}: ${setModelResult.stderr}`); ['--set-main', provider.model],
} { cwd: testDir }
);
if (setModelResult.exitCode !== 0) {
throw new Error(
`Failed to set model for ${provider.name}: ${setModelResult.stderr}`
);
}
// Step 2: Execute add-task with standard prompt // Step 2: Execute add-task with standard prompt
logger.info(`Adding task with ${provider.name}...`); logger.info(`Adding task with ${provider.name}...`);
const addTaskArgs = ['--prompt', standardPrompt]; const addTaskArgs = ['--prompt', standardPrompt];
if (provider.flags && provider.flags.length > 0) { if (provider.flags && provider.flags.length > 0) {
addTaskArgs.push(...provider.flags); addTaskArgs.push(...provider.flags);
} }
const addTaskResult = await helpers.taskMaster('add-task', addTaskArgs, { const addTaskResult = await helpers.taskMaster(
cwd: testDir, 'add-task',
timeout: 120000 // 2 minutes timeout for AI tasks addTaskArgs,
}); {
cwd: testDir,
timeout: 120000 // 2 minutes timeout for AI tasks
}
);
if (addTaskResult.exitCode !== 0) { if (addTaskResult.exitCode !== 0) {
throw new Error(`Add-task failed: ${addTaskResult.stderr}`); throw new Error(`Add-task failed: ${addTaskResult.stderr}`);
} }
// Step 3: Extract task ID from output // Step 3: Extract task ID from output
const taskId = helpers.extractTaskId(addTaskResult.stdout); const taskId = helpers.extractTaskId(addTaskResult.stdout);
if (!taskId) { if (!taskId) {
throw new Error(`Failed to extract task ID from output`); throw new Error(`Failed to extract task ID from output`);
} }
providerResult.taskId = taskId; providerResult.taskId = taskId;
logger.success(`✓ Created task ${taskId} with ${provider.name}`); logger.success(`✓ Created task ${taskId} with ${provider.name}`);
// Step 4: Get task details // Step 4: Get task details
const showResult = await helpers.taskMaster('show', [taskId], { cwd: testDir }); const showResult = await helpers.taskMaster('show', [taskId], {
if (showResult.exitCode === 0) { cwd: testDir
providerResult.taskDetails = showResult.stdout; });
if (showResult.exitCode === 0) {
providerResult.taskDetails = showResult.stdout;
// Analyze task features // Analyze task features
providerResult.features.hasTitle = showResult.stdout.includes('Title:') || providerResult.features.hasTitle =
showResult.stdout.includes('Task:'); showResult.stdout.includes('Title:') ||
providerResult.features.hasDescription = showResult.stdout.includes('Description:'); showResult.stdout.includes('Task:');
providerResult.features.hasSubtasks = showResult.stdout.includes('Subtasks:'); providerResult.features.hasDescription =
providerResult.features.hasDependencies = showResult.stdout.includes('Dependencies:'); showResult.stdout.includes('Description:');
providerResult.features.hasSubtasks =
showResult.stdout.includes('Subtasks:');
providerResult.features.hasDependencies =
showResult.stdout.includes('Dependencies:');
// Count subtasks // Count subtasks
const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g); const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g);
providerResult.subtaskCount = subtaskMatches ? subtaskMatches.length : 0; providerResult.subtaskCount = subtaskMatches
} ? subtaskMatches.length
: 0;
}
providerResult.status = 'success'; providerResult.status = 'success';
results.summary.successfulProviders++; results.summary.successfulProviders++;
} catch (error) {
providerResult.status = 'failed';
providerResult.error = error.message;
results.summary.failedProviders++;
logger.error(`${provider.name} test failed: ${error.message}`);
}
} catch (error) { providerResult.executionTime = Date.now() - startTime;
providerResult.status = 'failed'; totalExecutionTime += providerResult.executionTime;
providerResult.error = error.message;
results.summary.failedProviders++;
logger.error(`${provider.name} test failed: ${error.message}`);
}
providerResult.executionTime = Date.now() - startTime; results.providerComparison[provider.name] = providerResult;
totalExecutionTime += providerResult.executionTime; });
results.providerComparison[provider.name] = providerResult; // Wait for batch to complete
}); await Promise.all(batchPromises);
// Wait for batch to complete // Small delay between batches to avoid rate limits
await Promise.all(batchPromises); if (i + batchSize < providers.length) {
logger.info('Waiting 2 seconds before next batch...');
await helpers.wait(2000);
}
}
// Small delay between batches to avoid rate limits // Calculate summary statistics
if (i + batchSize < providers.length) { results.summary.averageExecutionTime = Math.round(
logger.info('Waiting 2 seconds before next batch...'); totalExecutionTime / providers.length
await helpers.wait(2000); );
} results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`;
}
// Calculate summary statistics // Log summary
results.summary.averageExecutionTime = Math.round(totalExecutionTime / providers.length); logger.info('\n=== Provider Test Summary ===');
results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`; logger.info(`Total providers tested: ${results.summary.totalProviders}`);
logger.info(`Successful: ${results.summary.successfulProviders}`);
logger.info(`Failed: ${results.summary.failedProviders}`);
logger.info(`Success rate: ${results.summary.successRate}`);
logger.info(
`Average execution time: ${results.summary.averageExecutionTime}ms`
);
// Log summary // Determine overall status
logger.info('\n=== Provider Test Summary ==='); if (results.summary.failedProviders === 0) {
logger.info(`Total providers tested: ${results.summary.totalProviders}`); logger.success('✅ All provider tests passed!');
logger.info(`Successful: ${results.summary.successfulProviders}`); } else if (results.summary.successfulProviders > 0) {
logger.info(`Failed: ${results.summary.failedProviders}`); results.status = 'partial';
logger.info(`Success rate: ${results.summary.successRate}`); logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`);
logger.info(`Average execution time: ${results.summary.averageExecutionTime}ms`); } else {
results.status = 'failed';
logger.error('❌ All provider tests failed');
}
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
// Determine overall status return results;
if (results.summary.failedProviders === 0) {
logger.success('✅ All provider tests passed!');
} else if (results.summary.successfulProviders > 0) {
results.status = 'partial';
logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`);
} else {
results.status = 'failed';
logger.error('❌ All provider tests failed');
}
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
return results;
} }

405
tests/e2e/tests/setup.test.js
View File

@@ -9,108 +9,119 @@ import { testConfig } from '../config/test-config.js';
* @returns {Promise<Object>} Test results object with status and directory path * @returns {Promise<Object>} Test results object with status and directory path
*/ */
export async function runSetupTest(logger, helpers) { export async function runSetupTest(logger, helpers) {
const testResults = { const testResults = {
status: 'pending', status: 'pending',
testDir: null, testDir: null,
steps: { steps: {
createDirectory: false, createDirectory: false,
linkGlobally: false, linkGlobally: false,
copyEnv: false, copyEnv: false,
initialize: false, initialize: false,
parsePrd: false, parsePrd: false,
analyzeComplexity: false, analyzeComplexity: false,
generateReport: false generateReport: false
}, },
errors: [], errors: [],
prdPath: null, prdPath: null,
complexityReport: null complexityReport: null
}; };
try { try {
// Step 1: Create test directory with timestamp // Step 1: Create test directory with timestamp
logger.step('Creating test directory'); logger.step('Creating test directory');
const timestamp = new Date().toISOString().replace(/[:.]/g, '').replace('T', '_').slice(0, -1); const timestamp = new Date()
const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`); .toISOString()
.replace(/[:.]/g, '')
.replace('T', '_')
.slice(0, -1);
const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`);
if (!existsSync(testDir)) { if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true }); mkdirSync(testDir, { recursive: true });
} }
testResults.testDir = testDir; testResults.testDir = testDir;
testResults.steps.createDirectory = true; testResults.steps.createDirectory = true;
logger.success(`Test directory created: ${testDir}`); logger.success(`Test directory created: ${testDir}`);
// Step 2: Link task-master globally // Step 2: Link task-master globally
logger.step('Linking task-master globally'); logger.step('Linking task-master globally');
const linkResult = await helpers.executeCommand('npm', ['link'], { const linkResult = await helpers.executeCommand('npm', ['link'], {
cwd: testConfig.paths.projectRoot, cwd: testConfig.paths.projectRoot,
timeout: 60000 timeout: 60000
}); });
if (linkResult.exitCode === 0) { if (linkResult.exitCode === 0) {
testResults.steps.linkGlobally = true; testResults.steps.linkGlobally = true;
logger.success('Task-master linked globally'); logger.success('Task-master linked globally');
} else { } else {
throw new Error(`Failed to link task-master: ${linkResult.stderr}`); throw new Error(`Failed to link task-master: ${linkResult.stderr}`);
} }
// Step 3: Copy .env file // Step 3: Copy .env file
logger.step('Copying .env file to test directory'); logger.step('Copying .env file to test directory');
const envSourcePath = testConfig.paths.mainEnvFile; const envSourcePath = testConfig.paths.mainEnvFile;
const envDestPath = join(testDir, '.env'); const envDestPath = join(testDir, '.env');
if (helpers.fileExists(envSourcePath)) { if (helpers.fileExists(envSourcePath)) {
if (helpers.copyFile(envSourcePath, envDestPath)) { if (helpers.copyFile(envSourcePath, envDestPath)) {
testResults.steps.copyEnv = true; testResults.steps.copyEnv = true;
logger.success('.env file copied successfully'); logger.success('.env file copied successfully');
} else { } else {
throw new Error('Failed to copy .env file'); throw new Error('Failed to copy .env file');
} }
} else { } else {
logger.warning('.env file not found at source, proceeding without it'); logger.warning('.env file not found at source, proceeding without it');
} }
// Step 4: Initialize project with task-master init // Step 4: Initialize project with task-master init
logger.step('Initializing project with task-master'); logger.step('Initializing project with task-master');
const initResult = await helpers.taskMaster('init', [ const initResult = await helpers.taskMaster(
'-y', 'init',
'--name="E2E Test ' + testDir.split('/').pop() + '"', [
'--description="Automated E2E test run"' '-y',
], { '--name="E2E Test ' + testDir.split('/').pop() + '"',
cwd: testDir, '--description="Automated E2E test run"'
timeout: 120000 ],
}); {
cwd: testDir,
timeout: 120000
}
);
if (initResult.exitCode === 0) { if (initResult.exitCode === 0) {
testResults.steps.initialize = true; testResults.steps.initialize = true;
logger.success('Project initialized successfully'); logger.success('Project initialized successfully');
// Save init debug log if available // Save init debug log if available
const initDebugPath = join(testDir, 'init-debug.log'); const initDebugPath = join(testDir, 'init-debug.log');
if (existsSync(initDebugPath)) { if (existsSync(initDebugPath)) {
logger.info('Init debug log saved'); logger.info('Init debug log saved');
} }
} else { } else {
throw new Error(`Initialization failed: ${initResult.stderr}`); throw new Error(`Initialization failed: ${initResult.stderr}`);
} }
// Step 5: Parse PRD from sample file // Step 5: Parse PRD from sample file
logger.step('Parsing PRD from sample file'); logger.step('Parsing PRD from sample file');
// First, copy the sample PRD to the test directory // First, copy the sample PRD to the test directory
const prdSourcePath = testConfig.paths.samplePrdSource; const prdSourcePath = testConfig.paths.samplePrdSource;
const prdDestPath = join(testDir, 'prd.txt'); const prdDestPath = join(testDir, 'prd.txt');
testResults.prdPath = prdDestPath; testResults.prdPath = prdDestPath;
if (!helpers.fileExists(prdSourcePath)) { if (!helpers.fileExists(prdSourcePath)) {
// If sample PRD doesn't exist in fixtures, use the example PRD // If sample PRD doesn't exist in fixtures, use the example PRD
const examplePrdPath = join(testConfig.paths.projectRoot, 'assets/example_prd.txt'); const examplePrdPath = join(
if (helpers.fileExists(examplePrdPath)) { testConfig.paths.projectRoot,
helpers.copyFile(examplePrdPath, prdDestPath); 'assets/example_prd.txt'
logger.info('Using example PRD file'); );
} else { if (helpers.fileExists(examplePrdPath)) {
// Create a minimal PRD for testing helpers.copyFile(examplePrdPath, prdDestPath);
const minimalPrd = `<PRD> logger.info('Using example PRD file');
} else {
// Create a minimal PRD for testing
const minimalPrd = `<PRD>
# Overview # Overview
A simple task management system for developers. A simple task management system for developers.
@@ -146,120 +157,138 @@ Phase 2: Enhanced features
6. Advanced features 6. Advanced features
</PRD>`; </PRD>`;
writeFileSync(prdDestPath, minimalPrd); writeFileSync(prdDestPath, minimalPrd);
logger.info('Created minimal PRD for testing'); logger.info('Created minimal PRD for testing');
} }
} else { } else {
helpers.copyFile(prdSourcePath, prdDestPath); helpers.copyFile(prdSourcePath, prdDestPath);
} }
// Parse the PRD // Parse the PRD
const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], { const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], {
cwd: testDir, cwd: testDir,
timeout: 180000 timeout: 180000
}); });
if (parsePrdResult.exitCode === 0) { if (parsePrdResult.exitCode === 0) {
testResults.steps.parsePrd = true; testResults.steps.parsePrd = true;
logger.success('PRD parsed successfully'); logger.success('PRD parsed successfully');
// Extract task count from output // Extract task count from output
const taskCountMatch = parsePrdResult.stdout.match(/(\d+) tasks? created/i); const taskCountMatch =
if (taskCountMatch) { parsePrdResult.stdout.match(/(\d+) tasks? created/i);
logger.info(`Created ${taskCountMatch[1]} tasks from PRD`); if (taskCountMatch) {
} logger.info(`Created ${taskCountMatch[1]} tasks from PRD`);
} else { }
throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`); } else {
} throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`);
}
// Step 6: Run complexity analysis // Step 6: Run complexity analysis
logger.step('Running complexity analysis on parsed tasks'); logger.step('Running complexity analysis on parsed tasks');
// Ensure reports directory exists // Ensure reports directory exists
const reportsDir = join(testDir, '.taskmaster/reports'); const reportsDir = join(testDir, '.taskmaster/reports');
if (!existsSync(reportsDir)) { if (!existsSync(reportsDir)) {
mkdirSync(reportsDir, { recursive: true }); mkdirSync(reportsDir, { recursive: true });
} }
const analyzeResult = await helpers.taskMaster('analyze-complexity', ['--research', '--output', '.taskmaster/reports/task-complexity-report.json'], { const analyzeResult = await helpers.taskMaster(
cwd: testDir, 'analyze-complexity',
timeout: 240000 [
}); '--research',
'--output',
'.taskmaster/reports/task-complexity-report.json'
],
{
cwd: testDir,
timeout: 240000
}
);
if (analyzeResult.exitCode === 0) { if (analyzeResult.exitCode === 0) {
testResults.steps.analyzeComplexity = true; testResults.steps.analyzeComplexity = true;
logger.success('Complexity analysis completed'); logger.success('Complexity analysis completed');
// Extract complexity information from output // Extract complexity information from output
const complexityMatch = analyzeResult.stdout.match(/Total Complexity Score: ([\d.]+)/); const complexityMatch = analyzeResult.stdout.match(
if (complexityMatch) { /Total Complexity Score: ([\d.]+)/
logger.info(`Total complexity score: ${complexityMatch[1]}`); );
} if (complexityMatch) {
} else { logger.info(`Total complexity score: ${complexityMatch[1]}`);
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`); }
} } else {
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`);
}
// Step 7: Generate complexity report // Step 7: Generate complexity report
logger.step('Generating complexity report'); logger.step('Generating complexity report');
const reportResult = await helpers.taskMaster('complexity-report', [], { const reportResult = await helpers.taskMaster('complexity-report', [], {
cwd: testDir, cwd: testDir,
timeout: 60000 timeout: 60000
}); });
if (reportResult.exitCode === 0) { if (reportResult.exitCode === 0) {
testResults.steps.generateReport = true; testResults.steps.generateReport = true;
logger.success('Complexity report generated'); logger.success('Complexity report generated');
// Check if complexity report file was created (not needed since complexity-report reads from the standard location) // Check if complexity report file was created (not needed since complexity-report reads from the standard location)
const reportPath = join(testDir, '.taskmaster/reports/task-complexity-report.json'); const reportPath = join(
if (helpers.fileExists(reportPath)) { testDir,
testResults.complexityReport = helpers.readJson(reportPath); '.taskmaster/reports/task-complexity-report.json'
logger.info('Complexity report saved to task-complexity-report.json'); );
if (helpers.fileExists(reportPath)) {
testResults.complexityReport = helpers.readJson(reportPath);
logger.info('Complexity report saved to task-complexity-report.json');
// Log summary if available // Log summary if available
if (testResults.complexityReport && testResults.complexityReport.summary) { if (
const summary = testResults.complexityReport.summary; testResults.complexityReport &&
logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`); testResults.complexityReport.summary
logger.info(`Average complexity: ${summary.averageComplexity || 0}`); ) {
} const summary = testResults.complexityReport.summary;
} logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`);
} else { logger.info(`Average complexity: ${summary.averageComplexity || 0}`);
logger.warning(`Complexity report generation had issues: ${reportResult.stderr}`); }
// Don't fail the test for report generation issues }
testResults.steps.generateReport = true; } else {
} logger.warning(
`Complexity report generation had issues: ${reportResult.stderr}`
);
// Don't fail the test for report generation issues
testResults.steps.generateReport = true;
}
// Verify tasks.json was created // Verify tasks.json was created
const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json'); const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json');
if (helpers.fileExists(tasksJsonPath)) { if (helpers.fileExists(tasksJsonPath)) {
const taskCount = helpers.getTaskCount(tasksJsonPath); const taskCount = helpers.getTaskCount(tasksJsonPath);
logger.info(`Verified tasks.json exists with ${taskCount} tasks`); logger.info(`Verified tasks.json exists with ${taskCount} tasks`);
} else { } else {
throw new Error('tasks.json was not created'); throw new Error('tasks.json was not created');
} }
// All steps completed successfully // All steps completed successfully
testResults.status = 'success'; testResults.status = 'success';
logger.success('Setup test completed successfully'); logger.success('Setup test completed successfully');
} catch (error) {
testResults.status = 'failed';
testResults.errors.push(error.message);
logger.error(`Setup test failed: ${error.message}`);
} catch (error) { // Log which steps completed
testResults.status = 'failed'; logger.info('Completed steps:');
testResults.errors.push(error.message); Object.entries(testResults.steps).forEach(([step, completed]) => {
logger.error(`Setup test failed: ${error.message}`); if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
// Log which steps completed // Flush logs before returning
logger.info('Completed steps:'); logger.flush();
Object.entries(testResults.steps).forEach(([step, completed]) => {
if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
// Flush logs before returning return testResults;
logger.flush();
return testResults;
} }
// Export default for direct execution // Export default for direct execution

399
tests/e2e/utils/error-handler.js
View File

@@ -3,234 +3,245 @@ import { join } from 'path';
import chalk from 'chalk'; import chalk from 'chalk';
export class ErrorHandler { export class ErrorHandler {
constructor(logger) { constructor(logger) {
this.logger = logger; this.logger = logger;
this.errors = []; this.errors = [];
this.warnings = []; this.warnings = [];
} }
/** /**
* Handle and categorize errors * Handle and categorize errors
*/ */
handleError(error, context = {}) { handleError(error, context = {}) {
const errorInfo = { const errorInfo = {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
message: error.message || 'Unknown error', message: error.message || 'Unknown error',
stack: error.stack, stack: error.stack,
context, context,
type: this.categorizeError(error) type: this.categorizeError(error)
}; };
this.errors.push(errorInfo); this.errors.push(errorInfo);
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`); this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
if (context.critical) { if (context.critical) {
throw error; throw error;
} }
return errorInfo; return errorInfo;
} }
/** /**
* Add a warning * Add a warning
*/ */
addWarning(message, context = {}) { addWarning(message, context = {}) {
const warning = { const warning = {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
message, message,
context context
}; };
this.warnings.push(warning); this.warnings.push(warning);
this.logger.warning(message); this.logger.warning(message);
} }
/** /**
* Categorize error types * Categorize error types
*/ */
categorizeError(error) { categorizeError(error) {
const message = error.message.toLowerCase(); const message = error.message.toLowerCase();
if (message.includes('command not found') || message.includes('not found')) { if (
return 'DEPENDENCY_ERROR'; message.includes('command not found') ||
} message.includes('not found')
if (message.includes('permission') || message.includes('access denied')) { ) {
return 'PERMISSION_ERROR'; return 'DEPENDENCY_ERROR';
} }
if (message.includes('timeout')) { if (message.includes('permission') || message.includes('access denied')) {
return 'TIMEOUT_ERROR'; return 'PERMISSION_ERROR';
} }
if (message.includes('api') || message.includes('rate limit')) { if (message.includes('timeout')) {
return 'API_ERROR'; return 'TIMEOUT_ERROR';
} }
if (message.includes('json') || message.includes('parse')) { if (message.includes('api') || message.includes('rate limit')) {
return 'PARSE_ERROR'; return 'API_ERROR';
} }
if (message.includes('file') || message.includes('directory')) { if (message.includes('json') || message.includes('parse')) {
return 'FILE_ERROR'; return 'PARSE_ERROR';
} }
if (message.includes('file') || message.includes('directory')) {
return 'FILE_ERROR';
}
return 'GENERAL_ERROR'; return 'GENERAL_ERROR';
} }
/** /**
* Get error summary * Get error summary
*/ */
getSummary() { getSummary() {
const errorsByType = {}; const errorsByType = {};
this.errors.forEach(error => { this.errors.forEach((error) => {
if (!errorsByType[error.type]) { if (!errorsByType[error.type]) {
errorsByType[error.type] = []; errorsByType[error.type] = [];
} }
errorsByType[error.type].push(error); errorsByType[error.type].push(error);
}); });
return { return {
totalErrors: this.errors.length, totalErrors: this.errors.length,
totalWarnings: this.warnings.length, totalWarnings: this.warnings.length,
errorsByType, errorsByType,
criticalErrors: this.errors.filter(e => e.context.critical), criticalErrors: this.errors.filter((e) => e.context.critical),
recentErrors: this.errors.slice(-5) recentErrors: this.errors.slice(-5)
}; };
} }
/** /**
* Generate error report * Generate error report
*/ */
generateReport(outputPath) { generateReport(outputPath) {
const summary = this.getSummary(); const summary = this.getSummary();
const report = { const report = {
generatedAt: new Date().toISOString(), generatedAt: new Date().toISOString(),
summary: { summary: {
totalErrors: summary.totalErrors, totalErrors: summary.totalErrors,
totalWarnings: summary.totalWarnings, totalWarnings: summary.totalWarnings,
errorTypes: Object.keys(summary.errorsByType) errorTypes: Object.keys(summary.errorsByType)
}, },
errors: this.errors, errors: this.errors,
warnings: this.warnings, warnings: this.warnings,
recommendations: this.generateRecommendations(summary) recommendations: this.generateRecommendations(summary)
}; };
writeFileSync(outputPath, JSON.stringify(report, null, 2)); writeFileSync(outputPath, JSON.stringify(report, null, 2));
return report; return report;
} }
/** /**
* Generate recommendations based on errors * Generate recommendations based on errors
*/ */
generateRecommendations(summary) { generateRecommendations(summary) {
const recommendations = []; const recommendations = [];
if (summary.errorsByType.DEPENDENCY_ERROR) { if (summary.errorsByType.DEPENDENCY_ERROR) {
recommendations.push({ recommendations.push({
type: 'DEPENDENCY', type: 'DEPENDENCY',
message: 'Install missing dependencies using npm install or check PATH', message: 'Install missing dependencies using npm install or check PATH',
errors: summary.errorsByType.DEPENDENCY_ERROR.length errors: summary.errorsByType.DEPENDENCY_ERROR.length
}); });
} }
if (summary.errorsByType.PERMISSION_ERROR) { if (summary.errorsByType.PERMISSION_ERROR) {
recommendations.push({ recommendations.push({
type: 'PERMISSION', type: 'PERMISSION',
message: 'Check file permissions or run with appropriate privileges', message: 'Check file permissions or run with appropriate privileges',
errors: summary.errorsByType.PERMISSION_ERROR.length errors: summary.errorsByType.PERMISSION_ERROR.length
}); });
} }
if (summary.errorsByType.API_ERROR) { if (summary.errorsByType.API_ERROR) {
recommendations.push({ recommendations.push({
type: 'API', type: 'API',
message: 'Check API keys, rate limits, or network connectivity', message: 'Check API keys, rate limits, or network connectivity',
errors: summary.errorsByType.API_ERROR.length errors: summary.errorsByType.API_ERROR.length
}); });
} }
if (summary.errorsByType.TIMEOUT_ERROR) { if (summary.errorsByType.TIMEOUT_ERROR) {
recommendations.push({ recommendations.push({
type: 'TIMEOUT', type: 'TIMEOUT',
message: 'Consider increasing timeout values or optimizing slow operations', message:
errors: summary.errorsByType.TIMEOUT_ERROR.length 'Consider increasing timeout values or optimizing slow operations',
}); errors: summary.errorsByType.TIMEOUT_ERROR.length
} });
}
return recommendations; return recommendations;
} }
/** /**
* Display error summary in console * Display error summary in console
*/ */
displaySummary() { displaySummary() {
const summary = this.getSummary(); const summary = this.getSummary();
if (summary.totalErrors === 0 && summary.totalWarnings === 0) { if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
console.log(chalk.green('✅ No errors or warnings detected')); console.log(chalk.green('✅ No errors or warnings detected'));
return; return;
} }
console.log(chalk.red.bold(`\n🚨 Error Summary:`)); console.log(chalk.red.bold(`\n🚨 Error Summary:`));
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`)); console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`)); console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
if (summary.totalErrors > 0) { if (summary.totalErrors > 0) {
console.log(chalk.red.bold('\n Error Types:')); console.log(chalk.red.bold('\n Error Types:'));
Object.entries(summary.errorsByType).forEach(([type, errors]) => { Object.entries(summary.errorsByType).forEach(([type, errors]) => {
console.log(chalk.red(` - ${type}: ${errors.length}`)); console.log(chalk.red(` - ${type}: ${errors.length}`));
}); });
if (summary.criticalErrors.length > 0) { if (summary.criticalErrors.length > 0) {
console.log(chalk.red.bold(`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`)); console.log(
summary.criticalErrors.forEach(error => { chalk.red.bold(
console.log(chalk.red(` - ${error.message}`)); `\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`
}); )
} );
} summary.criticalErrors.forEach((error) => {
console.log(chalk.red(` - ${error.message}`));
});
}
}
const recommendations = this.generateRecommendations(summary); const recommendations = this.generateRecommendations(summary);
if (recommendations.length > 0) { if (recommendations.length > 0) {
console.log(chalk.yellow.bold('\n💡 Recommendations:')); console.log(chalk.yellow.bold('\n💡 Recommendations:'));
recommendations.forEach(rec => { recommendations.forEach((rec) => {
console.log(chalk.yellow(` - ${rec.message}`)); console.log(chalk.yellow(` - ${rec.message}`));
}); });
} }
} }
/** /**
* Clear all errors and warnings * Clear all errors and warnings
*/ */
clear() { clear() {
this.errors = []; this.errors = [];
this.warnings = []; this.warnings = [];
} }
} }
/** /**
* Global error handler for uncaught exceptions * Global error handler for uncaught exceptions
*/ */
export function setupGlobalErrorHandlers(errorHandler, logger) { export function setupGlobalErrorHandlers(errorHandler, logger) {
process.on('uncaughtException', (error) => { process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`); logger.error(`Uncaught Exception: ${error.message}`);
errorHandler.handleError(error, { critical: true, source: 'uncaughtException' }); errorHandler.handleError(error, {
process.exit(1); critical: true,
}); source: 'uncaughtException'
});
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => { process.on('unhandledRejection', (reason, promise) => {
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`); logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
errorHandler.handleError(new Error(String(reason)), { errorHandler.handleError(new Error(String(reason)), {
critical: false, critical: false,
source: 'unhandledRejection' source: 'unhandledRejection'
}); });
}); });
process.on('SIGINT', () => { process.on('SIGINT', () => {
logger.info('\nReceived SIGINT, shutting down gracefully...'); logger.info('\nReceived SIGINT, shutting down gracefully...');
errorHandler.displaySummary(); errorHandler.displaySummary();
process.exit(130); process.exit(130);
}); });
process.on('SIGTERM', () => { process.on('SIGTERM', () => {
logger.info('\nReceived SIGTERM, shutting down...'); logger.info('\nReceived SIGTERM, shutting down...');
errorHandler.displaySummary(); errorHandler.displaySummary();
process.exit(143); process.exit(143);
}); });
} }

215
tests/e2e/utils/llm-analyzer.js
View File

@@ -2,56 +2,58 @@ import { readFileSync } from 'fs';
import fetch from 'node-fetch'; import fetch from 'node-fetch';
export class LLMAnalyzer { export class LLMAnalyzer {
constructor(config, logger) { constructor(config, logger) {
this.config = config; this.config = config;
this.logger = logger; this.logger = logger;
this.apiKey = process.env.ANTHROPIC_API_KEY; this.apiKey = process.env.ANTHROPIC_API_KEY;
this.apiEndpoint = 'https://api.anthropic.com/v1/messages'; this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
} }
async analyzeLog(logFile, providerSummaryFile = null) { async analyzeLog(logFile, providerSummaryFile = null) {
if (!this.config.llmAnalysis.enabled) { if (!this.config.llmAnalysis.enabled) {
this.logger.info('LLM analysis is disabled in configuration'); this.logger.info('LLM analysis is disabled in configuration');
return null; return null;
} }
if (!this.apiKey) { if (!this.apiKey) {
this.logger.error('ANTHROPIC_API_KEY not found in environment'); this.logger.error('ANTHROPIC_API_KEY not found in environment');
return null; return null;
} }
try { try {
const logContent = readFileSync(logFile, 'utf8'); const logContent = readFileSync(logFile, 'utf8');
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile); const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
const response = await this.callLLM(prompt); const response = await this.callLLM(prompt);
const analysis = this.parseResponse(response); const analysis = this.parseResponse(response);
// Calculate and log cost // Calculate and log cost
if (response.usage) { if (response.usage) {
const cost = this.calculateCost(response.usage); const cost = this.calculateCost(response.usage);
this.logger.addCost(cost); this.logger.addCost(cost);
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`); this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
} }
return analysis; return analysis;
} catch (error) { } catch (error) {
this.logger.error(`LLM analysis failed: ${error.message}`); this.logger.error(`LLM analysis failed: ${error.message}`);
return null; return null;
} }
} }
buildAnalysisPrompt(logContent, providerSummaryFile) { buildAnalysisPrompt(logContent, providerSummaryFile) {
let providerSummary = ''; let providerSummary = '';
if (providerSummaryFile) { if (providerSummaryFile) {
try { try {
providerSummary = readFileSync(providerSummaryFile, 'utf8'); providerSummary = readFileSync(providerSummaryFile, 'utf8');
} catch (error) { } catch (error) {
this.logger.warning(`Could not read provider summary file: ${error.message}`); this.logger.warning(
} `Could not read provider summary file: ${error.message}`
} );
}
}
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially. return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
Your goal is to: Your goal is to:
1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files). 1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files).
@@ -88,81 +90,82 @@ Return your analysis **strictly** in the following JSON format. Do not include a
Here is the main log content: Here is the main log content:
${logContent}`; ${logContent}`;
} }
async callLLM(prompt) { async callLLM(prompt) {
const payload = { const payload = {
model: this.config.llmAnalysis.model, model: this.config.llmAnalysis.model,
max_tokens: this.config.llmAnalysis.maxTokens, max_tokens: this.config.llmAnalysis.maxTokens,
messages: [ messages: [{ role: 'user', content: prompt }]
{ role: 'user', content: prompt } };
]
};
const response = await fetch(this.apiEndpoint, { const response = await fetch(this.apiEndpoint, {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'x-api-key': this.apiKey, 'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01' 'anthropic-version': '2023-06-01'
}, },
body: JSON.stringify(payload) body: JSON.stringify(payload)
}); });
if (!response.ok) { if (!response.ok) {
const error = await response.text(); const error = await response.text();
throw new Error(`LLM API call failed: ${response.status} - ${error}`); throw new Error(`LLM API call failed: ${response.status} - ${error}`);
} }
return response.json(); return response.json();
} }
parseResponse(response) { parseResponse(response) {
try { try {
const content = response.content[0].text; const content = response.content[0].text;
const jsonStart = content.indexOf('{'); const jsonStart = content.indexOf('{');
const jsonEnd = content.lastIndexOf('}'); const jsonEnd = content.lastIndexOf('}');
if (jsonStart === -1 || jsonEnd === -1) { if (jsonStart === -1 || jsonEnd === -1) {
throw new Error('No JSON found in response'); throw new Error('No JSON found in response');
} }
const jsonString = content.substring(jsonStart, jsonEnd + 1); const jsonString = content.substring(jsonStart, jsonEnd + 1);
return JSON.parse(jsonString); return JSON.parse(jsonString);
} catch (error) { } catch (error) {
this.logger.error(`Failed to parse LLM response: ${error.message}`); this.logger.error(`Failed to parse LLM response: ${error.message}`);
return null; return null;
} }
} }
calculateCost(usage) { calculateCost(usage) {
const modelCosts = { const modelCosts = {
'claude-3-7-sonnet-20250219': { 'claude-3-7-sonnet-20250219': {
input: 3.00, // per 1M tokens input: 3.0, // per 1M tokens
output: 15.00 // per 1M tokens output: 15.0 // per 1M tokens
} }
}; };
const costs = modelCosts[this.config.llmAnalysis.model] || { input: 0, output: 0 }; const costs = modelCosts[this.config.llmAnalysis.model] || {
const inputCost = (usage.input_tokens / 1000000) * costs.input; input: 0,
const outputCost = (usage.output_tokens / 1000000) * costs.output; output: 0
};
const inputCost = (usage.input_tokens / 1000000) * costs.input;
const outputCost = (usage.output_tokens / 1000000) * costs.output;
return inputCost + outputCost; return inputCost + outputCost;
} }
formatReport(analysis) { formatReport(analysis) {
if (!analysis) return null; if (!analysis) return null;
const report = { const report = {
title: 'TASKMASTER E2E Test Analysis Report', title: 'TASKMASTER E2E Test Analysis Report',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: analysis.overall_status, status: analysis.overall_status,
summary: analysis.llm_summary_points, summary: analysis.llm_summary_points,
verifiedSteps: analysis.verified_steps, verifiedSteps: analysis.verified_steps,
providerComparison: analysis.provider_add_task_comparison, providerComparison: analysis.provider_add_task_comparison,
issues: analysis.detected_issues issues: analysis.detected_issues
}; };
return report; return report;
} }
} }

205
tests/e2e/utils/logger.js
View File

@@ -3,122 +3,129 @@ import { join } from 'path';
import chalk from 'chalk'; import chalk from 'chalk';
export class TestLogger { export class TestLogger {
constructor(logDir, testRunId) { constructor(logDir, testRunId) {
this.logDir = logDir; this.logDir = logDir;
this.testRunId = testRunId; this.testRunId = testRunId;
this.startTime = Date.now(); this.startTime = Date.now();
this.stepCount = 0; this.stepCount = 0;
this.logFile = join(logDir, `e2e_run_${testRunId}.log`); this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
this.logBuffer = []; this.logBuffer = [];
this.totalCost = 0; this.totalCost = 0;
// Ensure log directory exists // Ensure log directory exists
if (!existsSync(logDir)) { if (!existsSync(logDir)) {
mkdirSync(logDir, { recursive: true }); mkdirSync(logDir, { recursive: true });
} }
} }
formatDuration(milliseconds) { formatDuration(milliseconds) {
const totalSeconds = Math.floor(milliseconds / 1000); const totalSeconds = Math.floor(milliseconds / 1000);
const minutes = Math.floor(totalSeconds / 60); const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60; const seconds = totalSeconds % 60;
return `${minutes}m${seconds.toString().padStart(2, '0')}s`; return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
} }
getElapsedTime() { getElapsedTime() {
return this.formatDuration(Date.now() - this.startTime); return this.formatDuration(Date.now() - this.startTime);
} }
formatLogEntry(level, message) { formatLogEntry(level, message) {
const timestamp = new Date().toISOString(); const timestamp = new Date().toISOString();
const elapsed = this.getElapsedTime(); const elapsed = this.getElapsedTime();
return `[${level}] [${elapsed}] ${timestamp} ${message}`; return `[${level}] [${elapsed}] ${timestamp} ${message}`;
} }
log(level, message, options = {}) { log(level, message, options = {}) {
const formattedMessage = this.formatLogEntry(level, message); const formattedMessage = this.formatLogEntry(level, message);
// Add to buffer // Add to buffer
this.logBuffer.push(formattedMessage); this.logBuffer.push(formattedMessage);
// Console output with colors // Console output with colors
let coloredMessage = formattedMessage; let coloredMessage = formattedMessage;
switch (level) { switch (level) {
case 'INFO': case 'INFO':
coloredMessage = chalk.blue(formattedMessage); coloredMessage = chalk.blue(formattedMessage);
break; break;
case 'SUCCESS': case 'SUCCESS':
coloredMessage = chalk.green(formattedMessage); coloredMessage = chalk.green(formattedMessage);
break; break;
case 'ERROR': case 'ERROR':
coloredMessage = chalk.red(formattedMessage); coloredMessage = chalk.red(formattedMessage);
break; break;
case 'WARNING': case 'WARNING':
coloredMessage = chalk.yellow(formattedMessage); coloredMessage = chalk.yellow(formattedMessage);
break; break;
} }
console.log(coloredMessage); console.log(coloredMessage);
// Write to file if immediate flush requested // Write to file if immediate flush requested
if (options.flush) { if (options.flush) {
this.flush(); this.flush();
} }
} }
info(message) { info(message) {
this.log('INFO', message); this.log('INFO', message);
} }
success(message) { success(message) {
this.log('SUCCESS', message); this.log('SUCCESS', message);
} }
error(message) { error(message) {
this.log('ERROR', message); this.log('ERROR', message);
} }
warning(message) { warning(message) {
this.log('WARNING', message); this.log('WARNING', message);
} }
step(message) { step(message) {
this.stepCount++; this.stepCount++;
const separator = '='.repeat(45); const separator = '='.repeat(45);
this.log('STEP', `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`); this.log(
} 'STEP',
`\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`
);
}
addCost(cost) { addCost(cost) {
if (typeof cost === 'number' && !isNaN(cost)) { if (typeof cost === 'number' && !isNaN(cost)) {
this.totalCost += cost; this.totalCost += cost;
} }
} }
extractAndAddCost(output) { extractAndAddCost(output) {
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g; const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
let match; let match;
while ((match = costRegex.exec(output)) !== null) { while ((match = costRegex.exec(output)) !== null) {
const cost = parseFloat(match[1]); const cost = parseFloat(match[1]);
this.addCost(cost); this.addCost(cost);
} }
} }
flush() { flush() {
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8'); writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
} }
getSummary() { getSummary() {
const duration = this.formatDuration(Date.now() - this.startTime); const duration = this.formatDuration(Date.now() - this.startTime);
const successCount = this.logBuffer.filter(line => line.includes('[SUCCESS]')).length; const successCount = this.logBuffer.filter((line) =>
const errorCount = this.logBuffer.filter(line => line.includes('[ERROR]')).length; line.includes('[SUCCESS]')
).length;
const errorCount = this.logBuffer.filter((line) =>
line.includes('[ERROR]')
).length;
return { return {
duration, duration,
totalSteps: this.stepCount, totalSteps: this.stepCount,
successCount, successCount,
errorCount, errorCount,
totalCost: this.totalCost.toFixed(6), totalCost: this.totalCost.toFixed(6),
logFile: this.logFile logFile: this.logFile
}; };
} }
} }

309
tests/e2e/utils/test-helpers.js
View File

@@ -3,185 +3,190 @@ import { readFileSync, existsSync, copyFileSync } from 'fs';
import { join } from 'path'; import { join } from 'path';
export class TestHelpers { export class TestHelpers {
constructor(logger) { constructor(logger) {
this.logger = logger; this.logger = logger;
} }
/** /**
* Execute a command and return output * Execute a command and return output
* @param {string} command - Command to execute * @param {string} command - Command to execute
* @param {string[]} args - Command arguments * @param {string[]} args - Command arguments
* @param {Object} options - Execution options * @param {Object} options - Execution options
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>} * @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
*/ */
async executeCommand(command, args = [], options = {}) { async executeCommand(command, args = [], options = {}) {
return new Promise((resolve) => { return new Promise((resolve) => {
const spawnOptions = { const spawnOptions = {
cwd: options.cwd || process.cwd(), cwd: options.cwd || process.cwd(),
env: { ...process.env, ...options.env }, env: { ...process.env, ...options.env },
shell: true shell: true
}; };
// When using shell: true, pass the full command as a single string // When using shell: true, pass the full command as a single string
const fullCommand = args.length > 0 ? `${command} ${args.join(' ')}` : command; const fullCommand =
const child = spawn(fullCommand, [], spawnOptions); args.length > 0 ? `${command} ${args.join(' ')}` : command;
let stdout = ''; const child = spawn(fullCommand, [], spawnOptions);
let stderr = ''; let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => { child.stdout.on('data', (data) => {
stdout += data.toString(); stdout += data.toString();
}); });
child.stderr.on('data', (data) => { child.stderr.on('data', (data) => {
stderr += data.toString(); stderr += data.toString();
}); });
child.on('close', (exitCode) => { child.on('close', (exitCode) => {
const output = stdout + stderr; const output = stdout + stderr;
// Extract and log costs // Extract and log costs
this.logger.extractAndAddCost(output); this.logger.extractAndAddCost(output);
resolve({ stdout, stderr, exitCode }); resolve({ stdout, stderr, exitCode });
}); });
// Handle timeout // Handle timeout
if (options.timeout) { if (options.timeout) {
setTimeout(() => { setTimeout(() => {
child.kill('SIGTERM'); child.kill('SIGTERM');
}, options.timeout); }, options.timeout);
} }
}); });
} }
/** /**
* Execute task-master command * Execute task-master command
* @param {string} subcommand - Task-master subcommand * @param {string} subcommand - Task-master subcommand
* @param {string[]} args - Command arguments * @param {string[]} args - Command arguments
* @param {Object} options - Execution options * @param {Object} options - Execution options
*/ */
async taskMaster(subcommand, args = [], options = {}) { async taskMaster(subcommand, args = [], options = {}) {
const fullArgs = [subcommand, ...args]; const fullArgs = [subcommand, ...args];
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`); this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
const result = await this.executeCommand('task-master', fullArgs, options); const result = await this.executeCommand('task-master', fullArgs, options);
if (result.exitCode !== 0 && !options.allowFailure) { if (result.exitCode !== 0 && !options.allowFailure) {
this.logger.error(`Command failed with exit code ${result.exitCode}`); this.logger.error(`Command failed with exit code ${result.exitCode}`);
this.logger.error(`stderr: ${result.stderr}`); this.logger.error(`stderr: ${result.stderr}`);
} }
return result; return result;
} }
/** /**
* Check if a file exists * Check if a file exists
*/ */
fileExists(filePath) { fileExists(filePath) {
return existsSync(filePath); return existsSync(filePath);
} }
/** /**
* Read JSON file * Read JSON file
*/ */
readJson(filePath) { readJson(filePath) {
try { try {
const content = readFileSync(filePath, 'utf8'); const content = readFileSync(filePath, 'utf8');
return JSON.parse(content); return JSON.parse(content);
} catch (error) { } catch (error) {
this.logger.error(`Failed to read JSON file ${filePath}: ${error.message}`); this.logger.error(
return null; `Failed to read JSON file ${filePath}: ${error.message}`
} );
} return null;
}
}
/** /**
* Copy file * Copy file
*/ */
copyFile(source, destination) { copyFile(source, destination) {
try { try {
copyFileSync(source, destination); copyFileSync(source, destination);
return true; return true;
} catch (error) { } catch (error) {
this.logger.error(`Failed to copy file from ${source} to ${destination}: ${error.message}`); this.logger.error(
return false; `Failed to copy file from ${source} to ${destination}: ${error.message}`
} );
} return false;
}
}
/** /**
* Wait for a specified duration * Wait for a specified duration
*/ */
async wait(milliseconds) { async wait(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds)); return new Promise((resolve) => setTimeout(resolve, milliseconds));
} }
/** /**
* Verify task exists in tasks.json * Verify task exists in tasks.json
*/ */
verifyTaskExists(tasksFile, taskId, tagName = 'master') { verifyTaskExists(tasksFile, taskId, tagName = 'master') {
const tasks = this.readJson(tasksFile); const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return false; if (!tasks || !tasks[tagName]) return false;
return tasks[tagName].tasks.some(task => task.id === taskId); return tasks[tagName].tasks.some((task) => task.id === taskId);
} }
/** /**
* Get task count for a tag * Get task count for a tag
*/ */
getTaskCount(tasksFile, tagName = 'master') { getTaskCount(tasksFile, tagName = 'master') {
const tasks = this.readJson(tasksFile); const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return 0; if (!tasks || !tasks[tagName]) return 0;
return tasks[tagName].tasks.length; return tasks[tagName].tasks.length;
} }
/** /**
* Extract task ID from command output * Extract task ID from command output
*/ */
extractTaskId(output) { extractTaskId(output) {
const patterns = [ const patterns = [
/✓ Added new task #(\d+(?:\.\d+)?)/, /✓ Added new task #(\d+(?:\.\d+)?)/,
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/, /✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
/Task (\d+(?:\.\d+)?) Created Successfully/ /Task (\d+(?:\.\d+)?) Created Successfully/
]; ];
for (const pattern of patterns) { for (const pattern of patterns) {
const match = output.match(pattern); const match = output.match(pattern);
if (match) { if (match) {
return match[1]; return match[1];
} }
} }
return null; return null;
} }
/** /**
* Run multiple async operations in parallel * Run multiple async operations in parallel
*/ */
async runParallel(operations) { async runParallel(operations) {
return Promise.all(operations); return Promise.all(operations);
} }
/** /**
* Run operations with concurrency limit * Run operations with concurrency limit
*/ */
async runWithConcurrency(operations, limit = 3) { async runWithConcurrency(operations, limit = 3) {
const results = []; const results = [];
const executing = []; const executing = [];
for (const operation of operations) { for (const operation of operations) {
const promise = operation().then(result => { const promise = operation().then((result) => {
executing.splice(executing.indexOf(promise), 1); executing.splice(executing.indexOf(promise), 1);
return result; return result;
}); });
results.push(promise); results.push(promise);
executing.push(promise); executing.push(promise);
if (executing.length >= limit) { if (executing.length >= limit) {
await Promise.race(executing); await Promise.race(executing);
} }
} }
return Promise.all(results); return Promise.all(results);
} }
} }