ros/claude-task-master
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: run format

Browse Source
This commit is contained in:
Ralph Khreish
2025-07-08 09:40:30 +03:00
parent 395693af24
commit d4208f372a
12 changed files with 2099 additions and 1776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
tests/e2e/config/test-config.js
View File

@@ -10,56 +10,63 @@ const projectRoot = join(__dirname, '../../..');
dotenvConfig({ path: join(projectRoot, '.env') }); dotenvConfig({ path: join(projectRoot, '.env') });
export const testConfig = { export const testConfig = {
// Paths // Paths
paths: { paths: {
projectRoot, projectRoot,
sourceDir: projectRoot, sourceDir: projectRoot,
baseTestDir: join(projectRoot, 'tests/e2e/_runs'), baseTestDir: join(projectRoot, 'tests/e2e/_runs'),
logDir: join(projectRoot, 'tests/e2e/log'), logDir: join(projectRoot, 'tests/e2e/log'),
samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'), samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'),
mainEnvFile: join(projectRoot, '.env'), mainEnvFile: join(projectRoot, '.env'),
supportedModelsFile: join(projectRoot, 'scripts/modules/supported-models.json') supportedModelsFile: join(
}, projectRoot,
'scripts/modules/supported-models.json'
)
},
// Test settings // Test settings
settings: { settings: {
runVerificationTest: true, runVerificationTest: true,
parallelTestGroups: 4, // Number of parallel test groups parallelTestGroups: 4, // Number of parallel test groups
timeout: 600000, // 10 minutes default timeout timeout: 600000, // 10 minutes default timeout
retryAttempts: 2 retryAttempts: 2
}, },
// Provider test configuration // Provider test configuration
providers: [ providers: [
{ name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] }, { name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] },
{ name: 'openai', model: 'gpt-4o', flags: [] }, { name: 'openai', model: 'gpt-4o', flags: [] },
{ name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] }, { name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] },
{ name: 'perplexity', model: 'sonar-pro', flags: [] }, { name: 'perplexity', model: 'sonar-pro', flags: [] },
{ name: 'xai', model: 'grok-3', flags: [] }, { name: 'xai', model: 'grok-3', flags: [] },
{ name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] } { name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] }
], ],
// Test prompts // Test prompts
prompts: { prompts: {
addTask: 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.', addTask:
updateTask: 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.', 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.',
updateFromTask: 'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.', updateTask:
updateSubtask: 'Implementation note: Remember to handle potential API errors and display a user-friendly message.' 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.',
}, updateFromTask:
'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.',
updateSubtask:
'Implementation note: Remember to handle potential API errors and display a user-friendly message.'
},
// LLM Analysis settings // LLM Analysis settings
llmAnalysis: { llmAnalysis: {
enabled: true, enabled: true,
model: 'claude-3-7-sonnet-20250219', model: 'claude-3-7-sonnet-20250219',
provider: 'anthropic', provider: 'anthropic',
maxTokens: 3072 maxTokens: 3072
} }
}; };
// Export test groups for parallel execution // Export test groups for parallel execution
export const testGroups = { export const testGroups = {
setup: ['setup'], setup: ['setup'],
core: ['core'], core: ['core'],
providers: ['providers'], providers: ['providers'],
advanced: ['advanced'] advanced: ['advanced']
}; };

7
tests/e2e/run-e2e-tests.js
View File

@@ -237,14 +237,15 @@ async function runTests(options) {
} }
// Check if we need to run setup (either explicitly requested or needed for other tests) // Check if we need to run setup (either explicitly requested or needed for other tests)
const needsSetup = testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0); const needsSetup =
testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0);
if (needsSetup) { if (needsSetup) {
// Always run setup if we need a test directory // Always run setup if we need a test directory
if (!testsToRun.setup) { if (!testsToRun.setup) {
logger.info('No test directory available, running setup automatically'); logger.info('No test directory available, running setup automatically');
} }
logger.step('Running setup tests'); logger.step('Running setup tests');
const setupRunner = new SequentialTestRunner(logger, helpers); const setupRunner = new SequentialTestRunner(logger, helpers);
const setupResults = await setupRunner.runTests(['setup'], {}); const setupResults = await setupRunner.runTests(['setup'], {});

370
tests/e2e/runners/parallel-runner.js
View File

@@ -7,205 +7,219 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename); const __dirname = dirname(__filename);
export class ParallelTestRunner extends EventEmitter { export class ParallelTestRunner extends EventEmitter {
constructor(logger) { constructor(logger) {
super(); super();
this.logger = logger; this.logger = logger;
this.workers = []; this.workers = [];
this.results = {}; this.results = {};
} }
/** /**
* Run test groups in parallel * Run test groups in parallel
* @param {Object} testGroups - Groups of tests to run * @param {Object} testGroups - Groups of tests to run
* @param {Object} sharedContext - Shared context for all tests * @param {Object} sharedContext - Shared context for all tests
* @returns {Promise<Object>} Combined results from all test groups * @returns {Promise<Object>} Combined results from all test groups
*/ */
async runTestGroups(testGroups, sharedContext) { async runTestGroups(testGroups, sharedContext) {
const groupNames = Object.keys(testGroups); const groupNames = Object.keys(testGroups);
const workerPromises = []; const workerPromises = [];
this.logger.info(`Starting parallel execution of ${groupNames.length} test groups`); this.logger.info(
`Starting parallel execution of ${groupNames.length} test groups`
);
for (const groupName of groupNames) { for (const groupName of groupNames) {
const workerPromise = this.runTestGroup(groupName, testGroups[groupName], sharedContext); const workerPromise = this.runTestGroup(
workerPromises.push(workerPromise); groupName,
} testGroups[groupName],
sharedContext
);
workerPromises.push(workerPromise);
}
// Wait for all workers to complete // Wait for all workers to complete
const results = await Promise.allSettled(workerPromises); const results = await Promise.allSettled(workerPromises);
// Process results // Process results
const combinedResults = { const combinedResults = {
overall: 'passed', overall: 'passed',
groups: {}, groups: {},
summary: { summary: {
totalGroups: groupNames.length, totalGroups: groupNames.length,
passedGroups: 0, passedGroups: 0,
failedGroups: 0, failedGroups: 0,
errors: [] errors: []
} }
}; };
results.forEach((result, index) => { results.forEach((result, index) => {
const groupName = groupNames[index]; const groupName = groupNames[index];
if (result.status === 'fulfilled') {
combinedResults.groups[groupName] = result.value;
if (result.value.status === 'passed') {
combinedResults.summary.passedGroups++;
} else {
combinedResults.summary.failedGroups++;
combinedResults.overall = 'failed';
}
} else {
combinedResults.groups[groupName] = {
status: 'failed',
error: result.reason.message || 'Unknown error'
};
combinedResults.summary.failedGroups++;
combinedResults.summary.errors.push({
group: groupName,
error: result.reason.message
});
combinedResults.overall = 'failed';
}
});
return combinedResults; if (result.status === 'fulfilled') {
} combinedResults.groups[groupName] = result.value;
if (result.value.status === 'passed') {
combinedResults.summary.passedGroups++;
} else {
combinedResults.summary.failedGroups++;
combinedResults.overall = 'failed';
}
} else {
combinedResults.groups[groupName] = {
status: 'failed',
error: result.reason.message || 'Unknown error'
};
combinedResults.summary.failedGroups++;
combinedResults.summary.errors.push({
group: groupName,
error: result.reason.message
});
combinedResults.overall = 'failed';
}
});
/** return combinedResults;
* Run a single test group in a worker thread }
*/
async runTestGroup(groupName, testModules, sharedContext) {
return new Promise((resolve, reject) => {
const workerPath = join(__dirname, 'test-worker.js');
const worker = new Worker(workerPath, {
workerData: {
groupName,
testModules,
sharedContext,
logDir: this.logger.logDir,
testRunId: this.logger.testRunId
}
});
this.workers.push(worker); /**
* Run a single test group in a worker thread
*/
async runTestGroup(groupName, testModules, sharedContext) {
return new Promise((resolve, reject) => {
const workerPath = join(__dirname, 'test-worker.js');
// Handle messages from worker const worker = new Worker(workerPath, {
worker.on('message', (message) => { workerData: {
if (message.type === 'log') { groupName,
const level = message.level.toLowerCase(); testModules,
if (typeof this.logger[level] === 'function') { sharedContext,
this.logger[level](message.message); logDir: this.logger.logDir,
} else { testRunId: this.logger.testRunId
// Fallback to info if the level doesn't exist }
this.logger.info(message.message); });
}
} else if (message.type === 'step') {
this.logger.step(message.message);
} else if (message.type === 'cost') {
this.logger.addCost(message.cost);
} else if (message.type === 'results') {
this.results[groupName] = message.results;
}
});
// Handle worker completion this.workers.push(worker);
worker.on('exit', (code) => {
this.workers = this.workers.filter(w => w !== worker);
if (code === 0) {
resolve(this.results[groupName] || { status: 'passed', group: groupName });
} else {
reject(new Error(`Worker for group ${groupName} exited with code ${code}`));
}
});
// Handle worker errors // Handle messages from worker
worker.on('error', (error) => { worker.on('message', (message) => {
this.workers = this.workers.filter(w => w !== worker); if (message.type === 'log') {
reject(error); const level = message.level.toLowerCase();
}); if (typeof this.logger[level] === 'function') {
this.logger[level](message.message);
} else {
// Fallback to info if the level doesn't exist
this.logger.info(message.message);
}
} else if (message.type === 'step') {
this.logger.step(message.message);
} else if (message.type === 'cost') {
this.logger.addCost(message.cost);
} else if (message.type === 'results') {
this.results[groupName] = message.results;
}
});
}); // Handle worker completion
} worker.on('exit', (code) => {
this.workers = this.workers.filter((w) => w !== worker);
/** if (code === 0) {
* Terminate all running workers resolve(
*/ this.results[groupName] || { status: 'passed', group: groupName }
async terminate() { );
const terminationPromises = this.workers.map(worker => } else {
worker.terminate().catch(err => reject(
this.logger.warning(`Failed to terminate worker: ${err.message}`) new Error(`Worker for group ${groupName} exited with code ${code}`)
) );
); }
});
await Promise.all(terminationPromises); // Handle worker errors
this.workers = []; worker.on('error', (error) => {
} this.workers = this.workers.filter((w) => w !== worker);
reject(error);
});
});
}
/**
* Terminate all running workers
*/
async terminate() {
const terminationPromises = this.workers.map((worker) =>
worker
.terminate()
.catch((err) =>
this.logger.warning(`Failed to terminate worker: ${err.message}`)
)
);
await Promise.all(terminationPromises);
this.workers = [];
}
} }
/** /**
* Sequential test runner for comparison or fallback * Sequential test runner for comparison or fallback
*/ */
export class SequentialTestRunner { export class SequentialTestRunner {
constructor(logger, helpers) { constructor(logger, helpers) {
this.logger = logger; this.logger = logger;
this.helpers = helpers; this.helpers = helpers;
} }
/** /**
* Run tests sequentially * Run tests sequentially
*/ */
async runTests(testModules, context) { async runTests(testModules, context) {
const results = { const results = {
overall: 'passed', overall: 'passed',
tests: {}, tests: {},
summary: { summary: {
totalTests: testModules.length, totalTests: testModules.length,
passedTests: 0, passedTests: 0,
failedTests: 0, failedTests: 0,
errors: [] errors: []
} }
}; };
for (const testModule of testModules) { for (const testModule of testModules) {
try { try {
this.logger.step(`Running ${testModule} tests`); this.logger.step(`Running ${testModule} tests`);
// Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`);
const { default: testFn } = await import(testPath);
// Run the test
const testResults = await testFn(this.logger, this.helpers, context);
results.tests[testModule] = testResults;
if (testResults.status === 'passed') {
results.summary.passedTests++;
} else {
results.summary.failedTests++;
results.overall = 'failed';
}
} catch (error) {
this.logger.error(`Failed to run ${testModule}: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
return results; // Dynamic import of test module
} const testPath = join(
} dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test
const testResults = await testFn(this.logger, this.helpers, context);
results.tests[testModule] = testResults;
if (testResults.status === 'passed') {
results.summary.passedTests++;
} else {
results.summary.failedTests++;
results.overall = 'failed';
}
} catch (error) {
this.logger.error(`Failed to run ${testModule}: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
return results;
}
}

217
tests/e2e/runners/test-worker.js
View File

@@ -9,124 +9,127 @@ const __dirname = dirname(__filename);
// Worker logger that sends messages to parent // Worker logger that sends messages to parent
class WorkerLogger extends TestLogger { class WorkerLogger extends TestLogger {
constructor(logDir, testRunId, groupName) { constructor(logDir, testRunId, groupName) {
super(logDir, `${testRunId}_${groupName}`); super(logDir, `${testRunId}_${groupName}`);
this.groupName = groupName; this.groupName = groupName;
} }
log(level, message, options = {}) { log(level, message, options = {}) {
super.log(level, message, options); super.log(level, message, options);
// Send log to parent
parentPort.postMessage({
type: 'log',
level: level.toLowerCase(),
message: `[${this.groupName}] ${message}`
});
}
step(message) { // Send log to parent
super.step(message); parentPort.postMessage({
type: 'log',
parentPort.postMessage({ level: level.toLowerCase(),
type: 'step', message: `[${this.groupName}] ${message}`
message: `[${this.groupName}] ${message}` });
}); }
}
addCost(cost) { step(message) {
super.addCost(cost); super.step(message);
parentPort.postMessage({ parentPort.postMessage({
type: 'cost', type: 'step',
cost message: `[${this.groupName}] ${message}`
}); });
} }
addCost(cost) {
super.addCost(cost);
parentPort.postMessage({
type: 'cost',
cost
});
}
} }
// Main worker execution // Main worker execution
async function runTestGroup() { async function runTestGroup() {
const { groupName, testModules, sharedContext, logDir, testRunId } = workerData; const { groupName, testModules, sharedContext, logDir, testRunId } =
workerData;
const logger = new WorkerLogger(logDir, testRunId, groupName);
const helpers = new TestHelpers(logger);
logger.info(`Worker started for test group: ${groupName}`);
const results = {
group: groupName,
status: 'passed',
tests: {},
errors: [],
startTime: Date.now()
};
try { const logger = new WorkerLogger(logDir, testRunId, groupName);
// Run each test module in the group const helpers = new TestHelpers(logger);
for (const testModule of testModules) {
try {
logger.info(`Running test: ${testModule}`);
// Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`);
const { default: testFn } = await import(testPath);
// Run the test with shared context
const testResults = await testFn(logger, helpers, sharedContext);
results.tests[testModule] = testResults;
if (testResults.status !== 'passed') {
results.status = 'failed';
if (testResults.errors) {
results.errors.push(...testResults.errors);
}
}
} catch (error) {
logger.error(`Test ${testModule} failed: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) {
logger.error(`Worker error: ${error.message}`);
results.status = 'failed';
results.errors.push({
group: groupName,
error: error.message,
stack: error.stack
});
}
results.endTime = Date.now(); logger.info(`Worker started for test group: ${groupName}`);
results.duration = results.endTime - results.startTime;
// Flush logs and get summary
logger.flush();
const summary = logger.getSummary();
results.summary = summary;
// Send results to parent const results = {
parentPort.postMessage({ group: groupName,
type: 'results', status: 'passed',
results tests: {},
}); errors: [],
startTime: Date.now()
};
logger.info(`Worker completed for test group: ${groupName}`); try {
// Run each test module in the group
for (const testModule of testModules) {
try {
logger.info(`Running test: ${testModule}`);
// Dynamic import of test module
const testPath = join(
dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test with shared context
const testResults = await testFn(logger, helpers, sharedContext);
results.tests[testModule] = testResults;
if (testResults.status !== 'passed') {
results.status = 'failed';
if (testResults.errors) {
results.errors.push(...testResults.errors);
}
}
} catch (error) {
logger.error(`Test ${testModule} failed: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) {
logger.error(`Worker error: ${error.message}`);
results.status = 'failed';
results.errors.push({
group: groupName,
error: error.message,
stack: error.stack
});
}
results.endTime = Date.now();
results.duration = results.endTime - results.startTime;
// Flush logs and get summary
logger.flush();
const summary = logger.getSummary();
results.summary = summary;
// Send results to parent
parentPort.postMessage({
type: 'results',
results
});
logger.info(`Worker completed for test group: ${groupName}`);
} }
// Run the test group // Run the test group
runTestGroup().catch(error => { runTestGroup().catch((error) => {
console.error('Worker fatal error:', error); console.error('Worker fatal error:', error);
process.exit(1); process.exit(1);
}); });

675
tests/e2e/tests/advanced.test.js
View File

@@ -1,316 +1,443 @@
export default async function testAdvanced(logger, helpers, context) { export default async function testAdvanced(logger, helpers, context) {
const { testDir } = context; const { testDir } = context;
logger.info('Starting advanced features tests...'); logger.info('Starting advanced features tests...');
const results = []; const results = [];
try { try {
// Test Tag Context Management // Test Tag Context Management
logger.info('Testing tag context management...'); logger.info('Testing tag context management...');
// Create new tag contexts
const tag1Result = await helpers.taskMaster('add-tag', ['feature-auth', '--description', 'Authentication feature'], { cwd: testDir });
results.push({
test: 'Create tag context - feature-auth',
passed: tag1Result.exitCode === 0,
output: tag1Result.stdout
});
const tag2Result = await helpers.taskMaster('add-tag', ['feature-api', '--description', 'API development'], { cwd: testDir }); // Create new tag contexts
results.push({ const tag1Result = await helpers.taskMaster(
test: 'Create tag context - feature-api', 'add-tag',
passed: tag2Result.exitCode === 0, ['feature-auth', '--description', 'Authentication feature'],
output: tag2Result.stdout { cwd: testDir }
}); );
results.push({
test: 'Create tag context - feature-auth',
passed: tag1Result.exitCode === 0,
output: tag1Result.stdout
});
// Add task to feature-auth tag const tag2Result = await helpers.taskMaster(
const task1Result = await helpers.taskMaster('add-task', ['--tag=feature-auth', '--prompt', 'Implement user authentication'], { cwd: testDir }); 'add-tag',
results.push({ ['feature-api', '--description', 'API development'],
test: 'Add task to feature-auth tag', { cwd: testDir }
passed: task1Result.exitCode === 0, );
output: task1Result.stdout results.push({
}); test: 'Create tag context - feature-api',
passed: tag2Result.exitCode === 0,
output: tag2Result.stdout
});
// Add task to feature-api tag // Add task to feature-auth tag
const task2Result = await helpers.taskMaster('add-task', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'], { cwd: testDir }); const task1Result = await helpers.taskMaster(
results.push({ 'add-task',
test: 'Add task to feature-api tag', ['--tag=feature-auth', '--prompt', 'Implement user authentication'],
passed: task2Result.exitCode === 0, { cwd: testDir }
output: task2Result.stdout );
}); results.push({
test: 'Add task to feature-auth tag',
passed: task1Result.exitCode === 0,
output: task1Result.stdout
});
// List all tag contexts // Add task to feature-api tag
const listTagsResult = await helpers.taskMaster('tags', [], { cwd: testDir }); const task2Result = await helpers.taskMaster(
results.push({ 'add-task',
test: 'List all tag contexts', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'],
passed: listTagsResult.exitCode === 0 && { cwd: testDir }
listTagsResult.stdout.includes('feature-auth') && );
listTagsResult.stdout.includes('feature-api'), results.push({
output: listTagsResult.stdout test: 'Add task to feature-api tag',
}); passed: task2Result.exitCode === 0,
output: task2Result.stdout
});
// List tasks in feature-auth tag // List all tag contexts
const taggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-auth'], { cwd: testDir }); const listTagsResult = await helpers.taskMaster('tags', [], {
results.push({ cwd: testDir
test: 'List tasks in feature-auth tag', });
passed: taggedTasksResult.exitCode === 0 && results.push({
taggedTasksResult.stdout.includes('Implement user authentication'), test: 'List all tag contexts',
output: taggedTasksResult.stdout passed:
}); listTagsResult.exitCode === 0 &&
listTagsResult.stdout.includes('feature-auth') &&
listTagsResult.stdout.includes('feature-api'),
output: listTagsResult.stdout
});
// Test Model Configuration // List tasks in feature-auth tag
logger.info('Testing model configuration...'); const taggedTasksResult = await helpers.taskMaster(
'list',
['--tag=feature-auth'],
{ cwd: testDir }
);
results.push({
test: 'List tasks in feature-auth tag',
passed:
taggedTasksResult.exitCode === 0 &&
taggedTasksResult.stdout.includes('Implement user authentication'),
output: taggedTasksResult.stdout
});
// Set main model // Test Model Configuration
const setMainModelResult = await helpers.taskMaster('models', ['--set-main', 'gpt-4'], { cwd: testDir }); logger.info('Testing model configuration...');
results.push({
test: 'Set main model',
passed: setMainModelResult.exitCode === 0,
output: setMainModelResult.stdout
});
// Set research model // Set main model
const setResearchModelResult = await helpers.taskMaster('models', ['--set-research', 'claude-3-sonnet'], { cwd: testDir }); const setMainModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Set research model', ['--set-main', 'gpt-4'],
passed: setResearchModelResult.exitCode === 0, { cwd: testDir }
output: setResearchModelResult.stdout );
}); results.push({
test: 'Set main model',
passed: setMainModelResult.exitCode === 0,
output: setMainModelResult.stdout
});
// Set fallback model // Set research model
const setFallbackModelResult = await helpers.taskMaster('models', ['--set-fallback', 'gpt-3.5-turbo'], { cwd: testDir }); const setResearchModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Set fallback model', ['--set-research', 'claude-3-sonnet'],
passed: setFallbackModelResult.exitCode === 0, { cwd: testDir }
output: setFallbackModelResult.stdout );
}); results.push({
test: 'Set research model',
passed: setResearchModelResult.exitCode === 0,
output: setResearchModelResult.stdout
});
// Verify model configuration // Set fallback model
const showModelsResult = await helpers.taskMaster('models', [], { cwd: testDir }); const setFallbackModelResult = await helpers.taskMaster(
results.push({ 'models',
test: 'Show model configuration', ['--set-fallback', 'gpt-3.5-turbo'],
passed: showModelsResult.exitCode === 0 && { cwd: testDir }
showModelsResult.stdout.includes('gpt-4') && );
showModelsResult.stdout.includes('claude-3-sonnet') && results.push({
showModelsResult.stdout.includes('gpt-3.5-turbo'), test: 'Set fallback model',
output: showModelsResult.stdout passed: setFallbackModelResult.exitCode === 0,
}); output: setFallbackModelResult.stdout
});
// Test Task Expansion // Verify model configuration
logger.info('Testing task expansion...'); const showModelsResult = await helpers.taskMaster('models', [], {
cwd: testDir
});
results.push({
test: 'Show model configuration',
passed:
showModelsResult.exitCode === 0 &&
showModelsResult.stdout.includes('gpt-4') &&
showModelsResult.stdout.includes('claude-3-sonnet') &&
showModelsResult.stdout.includes('gpt-3.5-turbo'),
output: showModelsResult.stdout
});
// Add task for expansion // Test Task Expansion
const expandTaskResult = await helpers.taskMaster('add-task', ['--prompt', 'Build REST API'], { cwd: testDir }); logger.info('Testing task expansion...');
const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/);
const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null;
results.push({
test: 'Add task for expansion',
passed: expandTaskResult.exitCode === 0 && expandTaskId !== null,
output: expandTaskResult.stdout
});
if (expandTaskId) { // Add task for expansion
// Single task expansion const expandTaskResult = await helpers.taskMaster(
const expandResult = await helpers.taskMaster('expand', [expandTaskId], { cwd: testDir }); 'add-task',
results.push({ ['--prompt', 'Build REST API'],
test: 'Expand single task', { cwd: testDir }
passed: expandResult.exitCode === 0 && expandResult.stdout.includes('subtasks'), );
output: expandResult.stdout const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/);
}); const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null;
// Verify expand worked results.push({
const afterExpandResult = await helpers.taskMaster('show', [expandTaskId], { cwd: testDir }); test: 'Add task for expansion',
results.push({ passed: expandTaskResult.exitCode === 0 && expandTaskId !== null,
test: 'Verify task expansion', output: expandTaskResult.stdout
passed: afterExpandResult.exitCode === 0 && afterExpandResult.stdout.includes('subtasks'), });
output: afterExpandResult.stdout
});
// Force expand (re-expand) if (expandTaskId) {
const forceExpandResult = await helpers.taskMaster('expand', [expandTaskId, '--force'], { cwd: testDir }); // Single task expansion
results.push({ const expandResult = await helpers.taskMaster('expand', [expandTaskId], {
test: 'Force expand task', cwd: testDir
passed: forceExpandResult.exitCode === 0, });
output: forceExpandResult.stdout results.push({
}); test: 'Expand single task',
} passed:
expandResult.exitCode === 0 &&
expandResult.stdout.includes('subtasks'),
output: expandResult.stdout
});
// Test Subtask Management // Verify expand worked
logger.info('Testing subtask management...'); const afterExpandResult = await helpers.taskMaster(
'show',
[expandTaskId],
{ cwd: testDir }
);
results.push({
test: 'Verify task expansion',
passed:
afterExpandResult.exitCode === 0 &&
afterExpandResult.stdout.includes('subtasks'),
output: afterExpandResult.stdout
});
// Add task for subtask testing // Force expand (re-expand)
const subtaskParentResult = await helpers.taskMaster('add-task', ['--prompt', 'Create user interface'], { cwd: testDir }); const forceExpandResult = await helpers.taskMaster(
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/); 'expand',
const parentTaskId = parentMatch ? parentMatch[1] : null; [expandTaskId, '--force'],
{ cwd: testDir }
);
results.push({
test: 'Force expand task',
passed: forceExpandResult.exitCode === 0,
output: forceExpandResult.stdout
});
}
if (parentTaskId) { // Test Subtask Management
// Add subtasks manually logger.info('Testing subtask management...');
const addSubtask1Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Design mockups'], { cwd: testDir });
results.push({
test: 'Add subtask - Design mockups',
passed: addSubtask1Result.exitCode === 0,
output: addSubtask1Result.stdout
});
const addSubtask2Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Implement components'], { cwd: testDir }); // Add task for subtask testing
results.push({ const subtaskParentResult = await helpers.taskMaster(
test: 'Add subtask - Implement components', 'add-task',
passed: addSubtask2Result.exitCode === 0, ['--prompt', 'Create user interface'],
output: addSubtask2Result.stdout { cwd: testDir }
}); );
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/);
const parentTaskId = parentMatch ? parentMatch[1] : null;
// List subtasks (use show command to see subtasks) if (parentTaskId) {
const listSubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); // Add subtasks manually
results.push({ const addSubtask1Result = await helpers.taskMaster(
test: 'List subtasks', 'add-subtask',
passed: listSubtasksResult.exitCode === 0 && ['--parent', parentTaskId, '--title', 'Design mockups'],
listSubtasksResult.stdout.includes('Design mockups') && { cwd: testDir }
listSubtasksResult.stdout.includes('Implement components'), );
output: listSubtasksResult.stdout results.push({
}); test: 'Add subtask - Design mockups',
passed: addSubtask1Result.exitCode === 0,
output: addSubtask1Result.stdout
});
// Update subtask const addSubtask2Result = await helpers.taskMaster(
const subtaskId = `${parentTaskId}.1`; 'add-subtask',
const updateSubtaskResult = await helpers.taskMaster('update-subtask', ['--id', subtaskId, '--prompt', 'Create detailed mockups'], { cwd: testDir }); ['--parent', parentTaskId, '--title', 'Implement components'],
results.push({ { cwd: testDir }
test: 'Update subtask', );
passed: updateSubtaskResult.exitCode === 0, results.push({
output: updateSubtaskResult.stdout test: 'Add subtask - Implement components',
}); passed: addSubtask2Result.exitCode === 0,
output: addSubtask2Result.stdout
});
// Remove subtask // List subtasks (use show command to see subtasks)
const removeSubtaskId = `${parentTaskId}.2`; const listSubtasksResult = await helpers.taskMaster(
const removeSubtaskResult = await helpers.taskMaster('remove-subtask', ['--id', removeSubtaskId], { cwd: testDir }); 'show',
results.push({ [parentTaskId],
test: 'Remove subtask', { cwd: testDir }
passed: removeSubtaskResult.exitCode === 0, );
output: removeSubtaskResult.stdout results.push({
}); test: 'List subtasks',
passed:
listSubtasksResult.exitCode === 0 &&
listSubtasksResult.stdout.includes('Design mockups') &&
listSubtasksResult.stdout.includes('Implement components'),
output: listSubtasksResult.stdout
});
// Verify subtask changes // Update subtask
const verifySubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); const subtaskId = `${parentTaskId}.1`;
results.push({ const updateSubtaskResult = await helpers.taskMaster(
test: 'Verify subtask changes', 'update-subtask',
passed: verifySubtasksResult.exitCode === 0 && ['--id', subtaskId, '--prompt', 'Create detailed mockups'],
verifySubtasksResult.stdout.includes('Create detailed mockups') && { cwd: testDir }
!verifySubtasksResult.stdout.includes('Implement components'), );
output: verifySubtasksResult.stdout results.push({
}); test: 'Update subtask',
passed: updateSubtaskResult.exitCode === 0,
output: updateSubtaskResult.stdout
});
// Clear all subtasks // Remove subtask
const clearSubtasksResult = await helpers.taskMaster('clear-subtasks', ['--id', parentTaskId], { cwd: testDir }); const removeSubtaskId = `${parentTaskId}.2`;
results.push({ const removeSubtaskResult = await helpers.taskMaster(
test: 'Clear all subtasks', 'remove-subtask',
passed: clearSubtasksResult.exitCode === 0, ['--id', removeSubtaskId],
output: clearSubtasksResult.stdout { cwd: testDir }
}); );
results.push({
test: 'Remove subtask',
passed: removeSubtaskResult.exitCode === 0,
output: removeSubtaskResult.stdout
});
// Verify subtasks cleared // Verify subtask changes
const verifyClearResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); const verifySubtasksResult = await helpers.taskMaster(
results.push({ 'show',
test: 'Verify subtasks cleared', [parentTaskId],
passed: verifyClearResult.exitCode === 0 && { cwd: testDir }
(!verifyClearResult.stdout.includes('Design mockups') && );
!verifyClearResult.stdout.includes('Create detailed mockups')), results.push({
output: verifyClearResult.stdout test: 'Verify subtask changes',
}); passed:
} verifySubtasksResult.exitCode === 0 &&
verifySubtasksResult.stdout.includes('Create detailed mockups') &&
!verifySubtasksResult.stdout.includes('Implement components'),
output: verifySubtasksResult.stdout
});
// Test Expand All // Clear all subtasks
logger.info('Testing expand all...'); const clearSubtasksResult = await helpers.taskMaster(
'clear-subtasks',
['--id', parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'Clear all subtasks',
passed: clearSubtasksResult.exitCode === 0,
output: clearSubtasksResult.stdout
});
// Add multiple tasks // Verify subtasks cleared
await helpers.taskMaster('add-task', ['--prompt', 'Task A for expand all'], { cwd: testDir }); const verifyClearResult = await helpers.taskMaster(
await helpers.taskMaster('add-task', ['--prompt', 'Task B for expand all'], { cwd: testDir }); 'show',
[parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'Verify subtasks cleared',
passed:
verifyClearResult.exitCode === 0 &&
!verifyClearResult.stdout.includes('Design mockups') &&
!verifyClearResult.stdout.includes('Create detailed mockups'),
output: verifyClearResult.stdout
});
}
const expandAllResult = await helpers.taskMaster('expand', ['--all'], { cwd: testDir }); // Test Expand All
results.push({ logger.info('Testing expand all...');
test: 'Expand all tasks',
passed: expandAllResult.exitCode === 0,
output: expandAllResult.stdout
});
// Test Generate Task Files // Add multiple tasks
logger.info('Testing generate task files...'); await helpers.taskMaster(
'add-task',
['--prompt', 'Task A for expand all'],
{ cwd: testDir }
);
await helpers.taskMaster(
'add-task',
['--prompt', 'Task B for expand all'],
{ cwd: testDir }
);
// Generate files for a specific task const expandAllResult = await helpers.taskMaster('expand', ['--all'], {
if (expandTaskId) { cwd: testDir
const generateResult = await helpers.taskMaster('generate', [expandTaskId], { cwd: testDir }); });
results.push({ results.push({
test: 'Generate task files', test: 'Expand all tasks',
passed: generateResult.exitCode === 0, passed: expandAllResult.exitCode === 0,
output: generateResult.stdout output: expandAllResult.stdout
}); });
// Check if files were created // Test Generate Task Files
const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`; logger.info('Testing generate task files...');
const fileExists = helpers.fileExists(taskFilePath);
results.push({
test: 'Verify generated task file exists',
passed: fileExists,
output: fileExists ? `Task file created at ${taskFilePath}` : 'Task file not found'
});
}
// Test Tag Context Integrity After Operations // Generate files for a specific task
logger.info('Testing tag context integrity after operations...'); if (expandTaskId) {
const generateResult = await helpers.taskMaster(
'generate',
[expandTaskId],
{ cwd: testDir }
);
results.push({
test: 'Generate task files',
passed: generateResult.exitCode === 0,
output: generateResult.stdout
});
// Verify tag contexts still exist // Check if files were created
const finalTagListResult = await helpers.taskMaster('tags', [], { cwd: testDir }); const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`;
results.push({ const fileExists = helpers.fileExists(taskFilePath);
test: 'Final tag context list verification',
passed: finalTagListResult.exitCode === 0 &&
finalTagListResult.stdout.includes('feature-auth') &&
finalTagListResult.stdout.includes('feature-api'),
output: finalTagListResult.stdout
});
// Verify tasks are still in their respective tag contexts results.push({
const finalTaggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-api'], { cwd: testDir }); test: 'Verify generated task file exists',
results.push({ passed: fileExists,
test: 'Final tasks in tag context verification', output: fileExists
passed: finalTaggedTasksResult.exitCode === 0 && ? `Task file created at ${taskFilePath}`
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'), : 'Task file not found'
output: finalTaggedTasksResult.stdout });
}); }
// Test Additional Advanced Features // Test Tag Context Integrity After Operations
logger.info('Testing additional advanced features...'); logger.info('Testing tag context integrity after operations...');
// Test priority task // Verify tag contexts still exist
const priorityTagResult = await helpers.taskMaster('add-task', ['--prompt', 'High priority task', '--priority', 'high'], { cwd: testDir }); const finalTagListResult = await helpers.taskMaster('tags', [], {
results.push({ cwd: testDir
test: 'Add task with high priority', });
passed: priorityTagResult.exitCode === 0, results.push({
output: priorityTagResult.stdout test: 'Final tag context list verification',
}); passed:
finalTagListResult.exitCode === 0 &&
finalTagListResult.stdout.includes('feature-auth') &&
finalTagListResult.stdout.includes('feature-api'),
output: finalTagListResult.stdout
});
// Test filtering by status // Verify tasks are still in their respective tag contexts
const statusFilterResult = await helpers.taskMaster('list', ['--status', 'pending'], { cwd: testDir }); const finalTaggedTasksResult = await helpers.taskMaster(
results.push({ 'list',
test: 'Filter by status', ['--tag=feature-api'],
passed: statusFilterResult.exitCode === 0, { cwd: testDir }
output: statusFilterResult.stdout );
}); results.push({
test: 'Final tasks in tag context verification',
passed:
finalTaggedTasksResult.exitCode === 0 &&
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'),
output: finalTaggedTasksResult.stdout
});
} catch (error) { // Test Additional Advanced Features
logger.error('Error in advanced features tests:', error); logger.info('Testing additional advanced features...');
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
const passed = results.filter(r => r.passed).length; // Test priority task
const total = results.length; const priorityTagResult = await helpers.taskMaster(
'add-task',
['--prompt', 'High priority task', '--priority', 'high'],
{ cwd: testDir }
);
results.push({
test: 'Add task with high priority',
passed: priorityTagResult.exitCode === 0,
output: priorityTagResult.stdout
});
return { // Test filtering by status
name: 'Advanced Features', const statusFilterResult = await helpers.taskMaster(
passed, 'list',
total, ['--status', 'pending'],
results, { cwd: testDir }
summary: `Advanced features tests: ${passed}/${total} passed` );
}; results.push({
}; test: 'Filter by status',
passed: statusFilterResult.exitCode === 0,
output: statusFilterResult.stdout
});
} catch (error) {
logger.error('Error in advanced features tests:', error);
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
const passed = results.filter((r) => r.passed).length;
const total = results.length;
return {
name: 'Advanced Features',
passed,
total,
results,
summary: `Advanced features tests: ${passed}/${total} passed`
};
}

598
tests/e2e/tests/core.test.js
View File

@@ -4,282 +4,376 @@
*/ */
export default async function testCoreOperations(logger, helpers, context) { export default async function testCoreOperations(logger, helpers, context) {
const { testDir } = context; const { testDir } = context;
const results = { const results = {
status: 'passed', status: 'passed',
errors: [] errors: []
}; };
try { try {
logger.info('Starting core task operations tests...'); logger.info('Starting core task operations tests...');
// Test 1: List tasks (may have tasks from PRD parsing) // Test 1: List tasks (may have tasks from PRD parsing)
logger.info('\nTest 1: List tasks'); logger.info('\nTest 1: List tasks');
const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir }); const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult1.exitCode !== 0) { if (listResult1.exitCode !== 0) {
throw new Error(`List command failed: ${listResult1.stderr}`); throw new Error(`List command failed: ${listResult1.stderr}`);
} }
// Check for expected output patterns - either empty or with tasks // Check for expected output patterns - either empty or with tasks
const hasValidOutput = listResult1.stdout.includes('No tasks found') || const hasValidOutput =
listResult1.stdout.includes('Task List') || listResult1.stdout.includes('No tasks found') ||
listResult1.stdout.includes('Project Dashboard') || listResult1.stdout.includes('Task List') ||
listResult1.stdout.includes('Listing tasks from'); listResult1.stdout.includes('Project Dashboard') ||
if (!hasValidOutput) { listResult1.stdout.includes('Listing tasks from');
throw new Error('Unexpected list output format'); if (!hasValidOutput) {
} throw new Error('Unexpected list output format');
logger.success('✓ List tasks successful'); }
logger.success('✓ List tasks successful');
// Test 2: Add manual task // Test 2: Add manual task
logger.info('\nTest 2: Add manual task'); logger.info('\nTest 2: Add manual task');
const addResult1 = await helpers.taskMaster('add-task', ['--title', 'Write unit tests', '--description', 'Create comprehensive unit tests for the application'], { cwd: testDir }); const addResult1 = await helpers.taskMaster(
if (addResult1.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add manual task: ${addResult1.stderr}`); [
} '--title',
const manualTaskId = helpers.extractTaskId(addResult1.stdout); 'Write unit tests',
if (!manualTaskId) { '--description',
throw new Error('Failed to extract task ID from add output'); 'Create comprehensive unit tests for the application'
} ],
logger.success(`✓ Added manual task with ID: ${manualTaskId}`); { cwd: testDir }
);
if (addResult1.exitCode !== 0) {
throw new Error(`Failed to add manual task: ${addResult1.stderr}`);
}
const manualTaskId = helpers.extractTaskId(addResult1.stdout);
if (!manualTaskId) {
throw new Error('Failed to extract task ID from add output');
}
logger.success(`✓ Added manual task with ID: ${manualTaskId}`);
// Test 3: Add AI task // Test 3: Add AI task
logger.info('\nTest 3: Add AI task'); logger.info('\nTest 3: Add AI task');
const addResult2 = await helpers.taskMaster('add-task', ['--prompt', 'Implement authentication system'], { cwd: testDir }); const addResult2 = await helpers.taskMaster(
if (addResult2.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add AI task: ${addResult2.stderr}`); ['--prompt', 'Implement authentication system'],
} { cwd: testDir }
const aiTaskId = helpers.extractTaskId(addResult2.stdout); );
if (!aiTaskId) { if (addResult2.exitCode !== 0) {
throw new Error('Failed to extract AI task ID from add output'); throw new Error(`Failed to add AI task: ${addResult2.stderr}`);
} }
logger.success(`✓ Added AI task with ID: ${aiTaskId}`); const aiTaskId = helpers.extractTaskId(addResult2.stdout);
if (!aiTaskId) {
throw new Error('Failed to extract AI task ID from add output');
}
logger.success(`✓ Added AI task with ID: ${aiTaskId}`);
// Test 4: Add another task for dependency testing // Test 4: Add another task for dependency testing
logger.info('\nTest 4: Add task for dependency testing'); logger.info('\nTest 4: Add task for dependency testing');
const addResult3 = await helpers.taskMaster('add-task', ['--title', 'Create database schema', '--description', 'Design and implement the database schema'], { cwd: testDir }); const addResult3 = await helpers.taskMaster(
if (addResult3.exitCode !== 0) { 'add-task',
throw new Error(`Failed to add database task: ${addResult3.stderr}`); [
} '--title',
const dbTaskId = helpers.extractTaskId(addResult3.stdout); 'Create database schema',
if (!dbTaskId) { '--description',
throw new Error('Failed to extract database task ID'); 'Design and implement the database schema'
} ],
logger.success(`✓ Added database task with ID: ${dbTaskId}`); { cwd: testDir }
);
if (addResult3.exitCode !== 0) {
throw new Error(`Failed to add database task: ${addResult3.stderr}`);
}
const dbTaskId = helpers.extractTaskId(addResult3.stdout);
if (!dbTaskId) {
throw new Error('Failed to extract database task ID');
}
logger.success(`✓ Added database task with ID: ${dbTaskId}`);
// Test 5: List tasks (should show our newly added tasks) // Test 5: List tasks (should show our newly added tasks)
logger.info('\nTest 5: List all tasks'); logger.info('\nTest 5: List all tasks');
const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir }); const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult2.exitCode !== 0) { if (listResult2.exitCode !== 0) {
throw new Error(`List command failed: ${listResult2.stderr}`); throw new Error(`List command failed: ${listResult2.stderr}`);
} }
// Check that we can find our task IDs in the output // Check that we can find our task IDs in the output
const hasTask11 = listResult2.stdout.includes('11'); const hasTask11 = listResult2.stdout.includes('11');
const hasTask12 = listResult2.stdout.includes('12'); const hasTask12 = listResult2.stdout.includes('12');
const hasTask13 = listResult2.stdout.includes('13'); const hasTask13 = listResult2.stdout.includes('13');
if (!hasTask11 || !hasTask12 || !hasTask13) {
throw new Error('Not all task IDs found in list output');
}
// Also check for partial matches (list may truncate titles)
const hasOurTasks = listResult2.stdout.includes('Write') ||
listResult2.stdout.includes('Create');
if (hasOurTasks) {
logger.success('✓ List tasks shows our added tasks');
} else {
logger.warning('Task titles may be truncated in list view');
}
// Test 6: Get next task if (!hasTask11 || !hasTask12 || !hasTask13) {
logger.info('\nTest 6: Get next task'); throw new Error('Not all task IDs found in list output');
const nextResult = await helpers.taskMaster('next', [], { cwd: testDir }); }
if (nextResult.exitCode !== 0) {
throw new Error(`Next task command failed: ${nextResult.stderr}`);
}
logger.success('✓ Get next task successful');
// Test 7: Show task details // Also check for partial matches (list may truncate titles)
logger.info('\nTest 7: Show task details'); const hasOurTasks =
const showResult = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); listResult2.stdout.includes('Write') ||
if (showResult.exitCode !== 0) { listResult2.stdout.includes('Create');
throw new Error(`Show task details failed: ${showResult.stderr}`); if (hasOurTasks) {
} logger.success('✓ List tasks shows our added tasks');
// Check that the task ID is shown and basic structure is present } else {
if (!showResult.stdout.includes(`Task: #${aiTaskId}`) && !showResult.stdout.includes(`ID: │ ${aiTaskId}`)) { logger.warning('Task titles may be truncated in list view');
throw new Error('Task ID not found in show output'); }
}
if (!showResult.stdout.includes('Status:') || !showResult.stdout.includes('Priority:')) {
throw new Error('Task details missing expected fields');
}
logger.success('✓ Show task details successful');
// Test 8: Add dependencies // Test 6: Get next task
logger.info('\nTest 8: Add dependencies'); logger.info('\nTest 6: Get next task');
const addDepResult = await helpers.taskMaster('add-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); const nextResult = await helpers.taskMaster('next', [], { cwd: testDir });
if (addDepResult.exitCode !== 0) { if (nextResult.exitCode !== 0) {
throw new Error(`Failed to add dependency: ${addDepResult.stderr}`); throw new Error(`Next task command failed: ${nextResult.stderr}`);
} }
logger.success('✓ Added dependency successfully'); logger.success('✓ Get next task successful');
// Test 9: Verify dependency was added // Test 7: Show task details
logger.info('\nTest 9: Verify dependency'); logger.info('\nTest 7: Show task details');
const showResult2 = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); const showResult = await helpers.taskMaster('show', [aiTaskId], {
if (showResult2.exitCode !== 0) { cwd: testDir
throw new Error(`Show task failed: ${showResult2.stderr}`); });
} if (showResult.exitCode !== 0) {
if (!showResult2.stdout.includes('Dependencies:') || !showResult2.stdout.includes(dbTaskId)) { throw new Error(`Show task details failed: ${showResult.stderr}`);
throw new Error('Dependency not shown in task details'); }
} // Check that the task ID is shown and basic structure is present
logger.success('✓ Dependency verified in task details'); if (
!showResult.stdout.includes(`Task: #${aiTaskId}`) &&
!showResult.stdout.includes(`ID: │ ${aiTaskId}`)
) {
throw new Error('Task ID not found in show output');
}
if (
!showResult.stdout.includes('Status:') ||
!showResult.stdout.includes('Priority:')
) {
throw new Error('Task details missing expected fields');
}
logger.success('✓ Show task details successful');
// Test 10: Test circular dependency (should fail) // Test 8: Add dependencies
logger.info('\nTest 10: Test circular dependency prevention'); logger.info('\nTest 8: Add dependencies');
const circularResult = await helpers.taskMaster('add-dependency', ['--id', dbTaskId, '--depends-on', aiTaskId], { const addDepResult = await helpers.taskMaster(
cwd: testDir, 'add-dependency',
allowFailure: true ['--id', aiTaskId, '--depends-on', dbTaskId],
}); { cwd: testDir }
if (circularResult.exitCode === 0) { );
throw new Error('Circular dependency was not prevented'); if (addDepResult.exitCode !== 0) {
} throw new Error(`Failed to add dependency: ${addDepResult.stderr}`);
if (!circularResult.stderr.toLowerCase().includes('circular')) { }
throw new Error('Expected circular dependency error message'); logger.success('✓ Added dependency successfully');
}
logger.success('✓ Circular dependency prevented successfully');
// Test 11: Test non-existent dependency // Test 9: Verify dependency was added
logger.info('\nTest 11: Test non-existent dependency'); logger.info('\nTest 9: Verify dependency');
const nonExistResult = await helpers.taskMaster('add-dependency', ['--id', '99999', '--depends-on', '88888'], { const showResult2 = await helpers.taskMaster('show', [aiTaskId], {
cwd: testDir, cwd: testDir
allowFailure: true });
}); if (showResult2.exitCode !== 0) {
if (nonExistResult.exitCode === 0) { throw new Error(`Show task failed: ${showResult2.stderr}`);
throw new Error('Non-existent dependency was incorrectly allowed'); }
} if (
logger.success('✓ Non-existent dependency handled correctly'); !showResult2.stdout.includes('Dependencies:') ||
!showResult2.stdout.includes(dbTaskId)
) {
throw new Error('Dependency not shown in task details');
}
logger.success('✓ Dependency verified in task details');
// Test 12: Remove dependency // Test 10: Test circular dependency (should fail)
logger.info('\nTest 12: Remove dependency'); logger.info('\nTest 10: Test circular dependency prevention');
const removeDepResult = await helpers.taskMaster('remove-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); const circularResult = await helpers.taskMaster(
if (removeDepResult.exitCode !== 0) { 'add-dependency',
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`); ['--id', dbTaskId, '--depends-on', aiTaskId],
} {
logger.success('✓ Removed dependency successfully'); cwd: testDir,
allowFailure: true
}
);
if (circularResult.exitCode === 0) {
throw new Error('Circular dependency was not prevented');
}
if (!circularResult.stderr.toLowerCase().includes('circular')) {
throw new Error('Expected circular dependency error message');
}
logger.success('✓ Circular dependency prevented successfully');
// Test 13: Validate dependencies // Test 11: Test non-existent dependency
logger.info('\nTest 13: Validate dependencies'); logger.info('\nTest 11: Test non-existent dependency');
const validateResult = await helpers.taskMaster('validate-dependencies', [], { cwd: testDir }); const nonExistResult = await helpers.taskMaster(
if (validateResult.exitCode !== 0) { 'add-dependency',
throw new Error(`Dependency validation failed: ${validateResult.stderr}`); ['--id', '99999', '--depends-on', '88888'],
} {
logger.success('✓ Dependency validation successful'); cwd: testDir,
allowFailure: true
}
);
if (nonExistResult.exitCode === 0) {
throw new Error('Non-existent dependency was incorrectly allowed');
}
logger.success('✓ Non-existent dependency handled correctly');
// Test 14: Update task description // Test 12: Remove dependency
logger.info('\nTest 14: Update task description'); logger.info('\nTest 12: Remove dependency');
const updateResult = await helpers.taskMaster('update-task', [manualTaskId, '--description', 'Write comprehensive unit tests'], { cwd: testDir }); const removeDepResult = await helpers.taskMaster(
if (updateResult.exitCode !== 0) { 'remove-dependency',
throw new Error(`Failed to update task: ${updateResult.stderr}`); ['--id', aiTaskId, '--depends-on', dbTaskId],
} { cwd: testDir }
logger.success('✓ Updated task description successfully'); );
if (removeDepResult.exitCode !== 0) {
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`);
}
logger.success('✓ Removed dependency successfully');
// Test 15: Add subtask // Test 13: Validate dependencies
logger.info('\nTest 15: Add subtask'); logger.info('\nTest 13: Validate dependencies');
const subtaskResult = await helpers.taskMaster('add-subtask', [manualTaskId, 'Write test for login'], { cwd: testDir }); const validateResult = await helpers.taskMaster(
if (subtaskResult.exitCode !== 0) { 'validate-dependencies',
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`); [],
} { cwd: testDir }
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1'; );
logger.success(`✓ Added subtask with ID: ${subtaskId}`); if (validateResult.exitCode !== 0) {
throw new Error(`Dependency validation failed: ${validateResult.stderr}`);
}
logger.success('✓ Dependency validation successful');
// Test 16: Verify subtask relationship // Test 14: Update task description
logger.info('\nTest 16: Verify subtask relationship'); logger.info('\nTest 14: Update task description');
const showResult3 = await helpers.taskMaster('show', [manualTaskId], { cwd: testDir }); const updateResult = await helpers.taskMaster(
if (showResult3.exitCode !== 0) { 'update-task',
throw new Error(`Show task failed: ${showResult3.stderr}`); [manualTaskId, '--description', 'Write comprehensive unit tests'],
} { cwd: testDir }
if (!showResult3.stdout.includes('Subtasks:')) { );
throw new Error('Subtasks section not shown in parent task'); if (updateResult.exitCode !== 0) {
} throw new Error(`Failed to update task: ${updateResult.stderr}`);
logger.success('✓ Subtask relationship verified'); }
logger.success('✓ Updated task description successfully');
// Test 17: Set task status to in_progress // Test 15: Add subtask
logger.info('\nTest 17: Set task status to in_progress'); logger.info('\nTest 15: Add subtask');
const statusResult1 = await helpers.taskMaster('set-status', [manualTaskId, 'in_progress'], { cwd: testDir }); const subtaskResult = await helpers.taskMaster(
if (statusResult1.exitCode !== 0) { 'add-subtask',
throw new Error(`Failed to update task status: ${statusResult1.stderr}`); [manualTaskId, 'Write test for login'],
} { cwd: testDir }
logger.success('✓ Set task status to in_progress'); );
if (subtaskResult.exitCode !== 0) {
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`);
}
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1';
logger.success(`✓ Added subtask with ID: ${subtaskId}`);
// Test 18: Set task status to completed // Test 16: Verify subtask relationship
logger.info('\nTest 18: Set task status to completed'); logger.info('\nTest 16: Verify subtask relationship');
const statusResult2 = await helpers.taskMaster('set-status', [dbTaskId, 'completed'], { cwd: testDir }); const showResult3 = await helpers.taskMaster('show', [manualTaskId], {
if (statusResult2.exitCode !== 0) { cwd: testDir
throw new Error(`Failed to complete task: ${statusResult2.stderr}`); });
} if (showResult3.exitCode !== 0) {
logger.success('✓ Set task status to completed'); throw new Error(`Show task failed: ${showResult3.stderr}`);
}
if (!showResult3.stdout.includes('Subtasks:')) {
throw new Error('Subtasks section not shown in parent task');
}
logger.success('✓ Subtask relationship verified');
// Test 19: List tasks with status filter // Test 17: Set task status to in_progress
logger.info('\nTest 19: List tasks by status'); logger.info('\nTest 17: Set task status to in_progress');
const listStatusResult = await helpers.taskMaster('list', ['--status', 'completed'], { cwd: testDir }); const statusResult1 = await helpers.taskMaster(
if (listStatusResult.exitCode !== 0) { 'set-status',
throw new Error(`List by status failed: ${listStatusResult.stderr}`); [manualTaskId, 'in_progress'],
} { cwd: testDir }
if (!listStatusResult.stdout.includes('Create database schema')) { );
throw new Error('Completed task not shown in filtered list'); if (statusResult1.exitCode !== 0) {
} throw new Error(`Failed to update task status: ${statusResult1.stderr}`);
logger.success('✓ List tasks by status successful'); }
logger.success('✓ Set task status to in_progress');
// Test 20: Remove single task // Test 18: Set task status to completed
logger.info('\nTest 20: Remove single task'); logger.info('\nTest 18: Set task status to completed');
const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], { cwd: testDir }); const statusResult2 = await helpers.taskMaster(
if (removeResult1.exitCode !== 0) { 'set-status',
throw new Error(`Failed to remove task: ${removeResult1.stderr}`); [dbTaskId, 'completed'],
} { cwd: testDir }
logger.success('✓ Removed single task successfully'); );
if (statusResult2.exitCode !== 0) {
throw new Error(`Failed to complete task: ${statusResult2.stderr}`);
}
logger.success('✓ Set task status to completed');
// Test 21: Remove multiple tasks // Test 19: List tasks with status filter
logger.info('\nTest 21: Remove multiple tasks'); logger.info('\nTest 19: List tasks by status');
const removeResult2 = await helpers.taskMaster('remove-task', [manualTaskId, aiTaskId], { cwd: testDir }); const listStatusResult = await helpers.taskMaster(
if (removeResult2.exitCode !== 0) { 'list',
throw new Error(`Failed to remove multiple tasks: ${removeResult2.stderr}`); ['--status', 'completed'],
} { cwd: testDir }
logger.success('✓ Removed multiple tasks successfully'); );
if (listStatusResult.exitCode !== 0) {
throw new Error(`List by status failed: ${listStatusResult.stderr}`);
}
if (!listStatusResult.stdout.includes('Create database schema')) {
throw new Error('Completed task not shown in filtered list');
}
logger.success('✓ List tasks by status successful');
// Test 22: Verify tasks were removed // Test 20: Remove single task
logger.info('\nTest 22: Verify tasks were removed'); logger.info('\nTest 20: Remove single task');
const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir }); const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], {
if (listResult3.exitCode !== 0) { cwd: testDir
throw new Error(`List command failed: ${listResult3.stderr}`); });
} if (removeResult1.exitCode !== 0) {
// Check that our specific task IDs are no longer in the list throw new Error(`Failed to remove task: ${removeResult1.stderr}`);
const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(listResult3.stdout); }
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(listResult3.stdout); logger.success('✓ Removed single task successfully');
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(listResult3.stdout);
if (stillHasTask11 || stillHasTask12 || stillHasTask13) {
throw new Error('Removed task IDs still appear in list');
}
logger.success('✓ Verified tasks were removed');
// Test 23: Fix dependencies (cleanup) // Test 21: Remove multiple tasks
logger.info('\nTest 23: Fix dependencies'); logger.info('\nTest 21: Remove multiple tasks');
const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], { cwd: testDir }); const removeResult2 = await helpers.taskMaster(
if (fixDepsResult.exitCode !== 0) { 'remove-task',
// Non-critical, just log [manualTaskId, aiTaskId],
logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`); { cwd: testDir }
} else { );
logger.success('✓ Fix dependencies command executed'); if (removeResult2.exitCode !== 0) {
} throw new Error(
`Failed to remove multiple tasks: ${removeResult2.stderr}`
);
}
logger.success('✓ Removed multiple tasks successfully');
logger.info('\n✅ All core task operations tests passed!'); // Test 22: Verify tasks were removed
logger.info('\nTest 22: Verify tasks were removed');
const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult3.exitCode !== 0) {
throw new Error(`List command failed: ${listResult3.stderr}`);
}
// Check that our specific task IDs are no longer in the list
const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(
listResult3.stdout
);
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(
listResult3.stdout
);
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(
listResult3.stdout
);
} catch (error) { if (stillHasTask11 || stillHasTask12 || stillHasTask13) {
results.status = 'failed'; throw new Error('Removed task IDs still appear in list');
results.errors.push({ }
test: 'core operations', logger.success('✓ Verified tasks were removed');
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
return results; // Test 23: Fix dependencies (cleanup)
} logger.info('\nTest 23: Fix dependencies');
const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], {
cwd: testDir
});
if (fixDepsResult.exitCode !== 0) {
// Non-critical, just log
logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`);
} else {
logger.success('✓ Fix dependencies command executed');
}
logger.info('\n✅ All core task operations tests passed!');
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'core operations',
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
return results;
}

302
tests/e2e/tests/providers.test.js
View File

@@ -4,161 +4,183 @@
*/ */
export default async function testProviders(logger, helpers, context) { export default async function testProviders(logger, helpers, context) {
const { testDir, config } = context; const { testDir, config } = context;
const results = { const results = {
status: 'passed', status: 'passed',
errors: [], errors: [],
providerComparison: {}, providerComparison: {},
summary: { summary: {
totalProviders: 0, totalProviders: 0,
successfulProviders: 0, successfulProviders: 0,
failedProviders: 0, failedProviders: 0,
averageExecutionTime: 0, averageExecutionTime: 0,
successRate: '0%' successRate: '0%'
} }
}; };
try { try {
logger.info('Starting multi-provider tests...'); logger.info('Starting multi-provider tests...');
const providers = config.providers; const providers = config.providers;
const standardPrompt = config.prompts.addTask; const standardPrompt = config.prompts.addTask;
results.summary.totalProviders = providers.length;
let totalExecutionTime = 0;
// Process providers in batches to avoid rate limits results.summary.totalProviders = providers.length;
const batchSize = 3; let totalExecutionTime = 0;
for (let i = 0; i < providers.length; i += batchSize) {
const batch = providers.slice(i, i + batchSize);
const batchPromises = batch.map(async (provider) => {
const providerResult = {
status: 'failed',
taskId: null,
executionTime: 0,
subtaskCount: 0,
features: {
hasTitle: false,
hasDescription: false,
hasSubtasks: false,
hasDependencies: false
},
error: null,
taskDetails: null
};
const startTime = Date.now(); // Process providers in batches to avoid rate limits
const batchSize = 3;
for (let i = 0; i < providers.length; i += batchSize) {
const batch = providers.slice(i, i + batchSize);
try { const batchPromises = batch.map(async (provider) => {
logger.info(`\nTesting provider: ${provider.name} with model: ${provider.model}`); const providerResult = {
status: 'failed',
taskId: null,
executionTime: 0,
subtaskCount: 0,
features: {
hasTitle: false,
hasDescription: false,
hasSubtasks: false,
hasDependencies: false
},
error: null,
taskDetails: null
};
// Step 1: Set the main model for this provider const startTime = Date.now();
logger.info(`Setting model to ${provider.model}...`);
const setModelResult = await helpers.taskMaster('models', ['--set-main', provider.model], { cwd: testDir });
if (setModelResult.exitCode !== 0) {
throw new Error(`Failed to set model for ${provider.name}: ${setModelResult.stderr}`);
}
// Step 2: Execute add-task with standard prompt try {
logger.info(`Adding task with ${provider.name}...`); logger.info(
const addTaskArgs = ['--prompt', standardPrompt]; `\nTesting provider: ${provider.name} with model: ${provider.model}`
if (provider.flags && provider.flags.length > 0) { );
addTaskArgs.push(...provider.flags);
}
const addTaskResult = await helpers.taskMaster('add-task', addTaskArgs, {
cwd: testDir,
timeout: 120000 // 2 minutes timeout for AI tasks
});
if (addTaskResult.exitCode !== 0) {
throw new Error(`Add-task failed: ${addTaskResult.stderr}`);
}
// Step 3: Extract task ID from output // Step 1: Set the main model for this provider
const taskId = helpers.extractTaskId(addTaskResult.stdout); logger.info(`Setting model to ${provider.model}...`);
if (!taskId) { const setModelResult = await helpers.taskMaster(
throw new Error(`Failed to extract task ID from output`); 'models',
} ['--set-main', provider.model],
providerResult.taskId = taskId; { cwd: testDir }
logger.success(`✓ Created task ${taskId} with ${provider.name}`); );
if (setModelResult.exitCode !== 0) {
throw new Error(
`Failed to set model for ${provider.name}: ${setModelResult.stderr}`
);
}
// Step 4: Get task details // Step 2: Execute add-task with standard prompt
const showResult = await helpers.taskMaster('show', [taskId], { cwd: testDir }); logger.info(`Adding task with ${provider.name}...`);
if (showResult.exitCode === 0) { const addTaskArgs = ['--prompt', standardPrompt];
providerResult.taskDetails = showResult.stdout; if (provider.flags && provider.flags.length > 0) {
addTaskArgs.push(...provider.flags);
// Analyze task features }
providerResult.features.hasTitle = showResult.stdout.includes('Title:') ||
showResult.stdout.includes('Task:');
providerResult.features.hasDescription = showResult.stdout.includes('Description:');
providerResult.features.hasSubtasks = showResult.stdout.includes('Subtasks:');
providerResult.features.hasDependencies = showResult.stdout.includes('Dependencies:');
// Count subtasks
const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g);
providerResult.subtaskCount = subtaskMatches ? subtaskMatches.length : 0;
}
providerResult.status = 'success'; const addTaskResult = await helpers.taskMaster(
results.summary.successfulProviders++; 'add-task',
addTaskArgs,
{
cwd: testDir,
timeout: 120000 // 2 minutes timeout for AI tasks
}
);
} catch (error) { if (addTaskResult.exitCode !== 0) {
providerResult.status = 'failed'; throw new Error(`Add-task failed: ${addTaskResult.stderr}`);
providerResult.error = error.message; }
results.summary.failedProviders++;
logger.error(`${provider.name} test failed: ${error.message}`);
}
providerResult.executionTime = Date.now() - startTime; // Step 3: Extract task ID from output
totalExecutionTime += providerResult.executionTime; const taskId = helpers.extractTaskId(addTaskResult.stdout);
if (!taskId) {
results.providerComparison[provider.name] = providerResult; throw new Error(`Failed to extract task ID from output`);
}); }
providerResult.taskId = taskId;
logger.success(`✓ Created task ${taskId} with ${provider.name}`);
// Wait for batch to complete // Step 4: Get task details
await Promise.all(batchPromises); const showResult = await helpers.taskMaster('show', [taskId], {
cwd: testDir
// Small delay between batches to avoid rate limits });
if (i + batchSize < providers.length) { if (showResult.exitCode === 0) {
logger.info('Waiting 2 seconds before next batch...'); providerResult.taskDetails = showResult.stdout;
await helpers.wait(2000);
}
}
// Calculate summary statistics // Analyze task features
results.summary.averageExecutionTime = Math.round(totalExecutionTime / providers.length); providerResult.features.hasTitle =
results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`; showResult.stdout.includes('Title:') ||
showResult.stdout.includes('Task:');
providerResult.features.hasDescription =
showResult.stdout.includes('Description:');
providerResult.features.hasSubtasks =
showResult.stdout.includes('Subtasks:');
providerResult.features.hasDependencies =
showResult.stdout.includes('Dependencies:');
// Log summary // Count subtasks
logger.info('\n=== Provider Test Summary ==='); const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g);
logger.info(`Total providers tested: ${results.summary.totalProviders}`); providerResult.subtaskCount = subtaskMatches
logger.info(`Successful: ${results.summary.successfulProviders}`); ? subtaskMatches.length
logger.info(`Failed: ${results.summary.failedProviders}`); : 0;
logger.info(`Success rate: ${results.summary.successRate}`); }
logger.info(`Average execution time: ${results.summary.averageExecutionTime}ms`);
// Determine overall status providerResult.status = 'success';
if (results.summary.failedProviders === 0) { results.summary.successfulProviders++;
logger.success('✅ All provider tests passed!'); } catch (error) {
} else if (results.summary.successfulProviders > 0) { providerResult.status = 'failed';
results.status = 'partial'; providerResult.error = error.message;
logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`); results.summary.failedProviders++;
} else { logger.error(`${provider.name} test failed: ${error.message}`);
results.status = 'failed'; }
logger.error('❌ All provider tests failed');
}
} catch (error) { providerResult.executionTime = Date.now() - startTime;
results.status = 'failed'; totalExecutionTime += providerResult.executionTime;
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
return results; results.providerComparison[provider.name] = providerResult;
} });
// Wait for batch to complete
await Promise.all(batchPromises);
// Small delay between batches to avoid rate limits
if (i + batchSize < providers.length) {
logger.info('Waiting 2 seconds before next batch...');
await helpers.wait(2000);
}
}
// Calculate summary statistics
results.summary.averageExecutionTime = Math.round(
totalExecutionTime / providers.length
);
results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`;
// Log summary
logger.info('\n=== Provider Test Summary ===');
logger.info(`Total providers tested: ${results.summary.totalProviders}`);
logger.info(`Successful: ${results.summary.successfulProviders}`);
logger.info(`Failed: ${results.summary.failedProviders}`);
logger.info(`Success rate: ${results.summary.successRate}`);
logger.info(
`Average execution time: ${results.summary.averageExecutionTime}ms`
);
// Determine overall status
if (results.summary.failedProviders === 0) {
logger.success('✅ All provider tests passed!');
} else if (results.summary.successfulProviders > 0) {
results.status = 'partial';
logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`);
} else {
results.status = 'failed';
logger.error('❌ All provider tests failed');
}
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
return results;
}

429
tests/e2e/tests/setup.test.js
View File

@@ -9,108 +9,119 @@ import { testConfig } from '../config/test-config.js';
* @returns {Promise<Object>} Test results object with status and directory path * @returns {Promise<Object>} Test results object with status and directory path
*/ */
export async function runSetupTest(logger, helpers) { export async function runSetupTest(logger, helpers) {
const testResults = { const testResults = {
status: 'pending', status: 'pending',
testDir: null, testDir: null,
steps: { steps: {
createDirectory: false, createDirectory: false,
linkGlobally: false, linkGlobally: false,
copyEnv: false, copyEnv: false,
initialize: false, initialize: false,
parsePrd: false, parsePrd: false,
analyzeComplexity: false, analyzeComplexity: false,
generateReport: false generateReport: false
}, },
errors: [], errors: [],
prdPath: null, prdPath: null,
complexityReport: null complexityReport: null
}; };
try { try {
// Step 1: Create test directory with timestamp // Step 1: Create test directory with timestamp
logger.step('Creating test directory'); logger.step('Creating test directory');
const timestamp = new Date().toISOString().replace(/[:.]/g, '').replace('T', '_').slice(0, -1); const timestamp = new Date()
const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`); .toISOString()
.replace(/[:.]/g, '')
if (!existsSync(testDir)) { .replace('T', '_')
mkdirSync(testDir, { recursive: true }); .slice(0, -1);
} const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`);
testResults.testDir = testDir;
testResults.steps.createDirectory = true;
logger.success(`Test directory created: ${testDir}`);
// Step 2: Link task-master globally if (!existsSync(testDir)) {
logger.step('Linking task-master globally'); mkdirSync(testDir, { recursive: true });
const linkResult = await helpers.executeCommand('npm', ['link'], { }
cwd: testConfig.paths.projectRoot,
timeout: 60000
});
if (linkResult.exitCode === 0) { testResults.testDir = testDir;
testResults.steps.linkGlobally = true; testResults.steps.createDirectory = true;
logger.success('Task-master linked globally'); logger.success(`Test directory created: ${testDir}`);
} else {
throw new Error(`Failed to link task-master: ${linkResult.stderr}`);
}
// Step 3: Copy .env file // Step 2: Link task-master globally
logger.step('Copying .env file to test directory'); logger.step('Linking task-master globally');
const envSourcePath = testConfig.paths.mainEnvFile; const linkResult = await helpers.executeCommand('npm', ['link'], {
const envDestPath = join(testDir, '.env'); cwd: testConfig.paths.projectRoot,
timeout: 60000
if (helpers.fileExists(envSourcePath)) { });
if (helpers.copyFile(envSourcePath, envDestPath)) {
testResults.steps.copyEnv = true;
logger.success('.env file copied successfully');
} else {
throw new Error('Failed to copy .env file');
}
} else {
logger.warning('.env file not found at source, proceeding without it');
}
// Step 4: Initialize project with task-master init if (linkResult.exitCode === 0) {
logger.step('Initializing project with task-master'); testResults.steps.linkGlobally = true;
const initResult = await helpers.taskMaster('init', [ logger.success('Task-master linked globally');
'-y', } else {
'--name="E2E Test ' + testDir.split('/').pop() + '"', throw new Error(`Failed to link task-master: ${linkResult.stderr}`);
'--description="Automated E2E test run"' }
], {
cwd: testDir,
timeout: 120000
});
if (initResult.exitCode === 0) { // Step 3: Copy .env file
testResults.steps.initialize = true; logger.step('Copying .env file to test directory');
logger.success('Project initialized successfully'); const envSourcePath = testConfig.paths.mainEnvFile;
const envDestPath = join(testDir, '.env');
// Save init debug log if available
const initDebugPath = join(testDir, 'init-debug.log');
if (existsSync(initDebugPath)) {
logger.info('Init debug log saved');
}
} else {
throw new Error(`Initialization failed: ${initResult.stderr}`);
}
// Step 5: Parse PRD from sample file if (helpers.fileExists(envSourcePath)) {
logger.step('Parsing PRD from sample file'); if (helpers.copyFile(envSourcePath, envDestPath)) {
testResults.steps.copyEnv = true;
// First, copy the sample PRD to the test directory logger.success('.env file copied successfully');
const prdSourcePath = testConfig.paths.samplePrdSource; } else {
const prdDestPath = join(testDir, 'prd.txt'); throw new Error('Failed to copy .env file');
testResults.prdPath = prdDestPath; }
} else {
if (!helpers.fileExists(prdSourcePath)) { logger.warning('.env file not found at source, proceeding without it');
// If sample PRD doesn't exist in fixtures, use the example PRD }
const examplePrdPath = join(testConfig.paths.projectRoot, 'assets/example_prd.txt');
if (helpers.fileExists(examplePrdPath)) { // Step 4: Initialize project with task-master init
helpers.copyFile(examplePrdPath, prdDestPath); logger.step('Initializing project with task-master');
logger.info('Using example PRD file'); const initResult = await helpers.taskMaster(
} else { 'init',
// Create a minimal PRD for testing [
const minimalPrd = `<PRD> '-y',
'--name="E2E Test ' + testDir.split('/').pop() + '"',
'--description="Automated E2E test run"'
],
{
cwd: testDir,
timeout: 120000
}
);
if (initResult.exitCode === 0) {
testResults.steps.initialize = true;
logger.success('Project initialized successfully');
// Save init debug log if available
const initDebugPath = join(testDir, 'init-debug.log');
if (existsSync(initDebugPath)) {
logger.info('Init debug log saved');
}
} else {
throw new Error(`Initialization failed: ${initResult.stderr}`);
}
// Step 5: Parse PRD from sample file
logger.step('Parsing PRD from sample file');
// First, copy the sample PRD to the test directory
const prdSourcePath = testConfig.paths.samplePrdSource;
const prdDestPath = join(testDir, 'prd.txt');
testResults.prdPath = prdDestPath;
if (!helpers.fileExists(prdSourcePath)) {
// If sample PRD doesn't exist in fixtures, use the example PRD
const examplePrdPath = join(
testConfig.paths.projectRoot,
'assets/example_prd.txt'
);
if (helpers.fileExists(examplePrdPath)) {
helpers.copyFile(examplePrdPath, prdDestPath);
logger.info('Using example PRD file');
} else {
// Create a minimal PRD for testing
const minimalPrd = `<PRD>
# Overview # Overview
A simple task management system for developers. A simple task management system for developers.
@@ -145,122 +156,140 @@ Phase 2: Enhanced features
5. CLI interface 5. CLI interface
6. Advanced features 6. Advanced features
</PRD>`; </PRD>`;
writeFileSync(prdDestPath, minimalPrd);
logger.info('Created minimal PRD for testing');
}
} else {
helpers.copyFile(prdSourcePath, prdDestPath);
}
// Parse the PRD writeFileSync(prdDestPath, minimalPrd);
const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], { logger.info('Created minimal PRD for testing');
cwd: testDir, }
timeout: 180000 } else {
}); helpers.copyFile(prdSourcePath, prdDestPath);
}
if (parsePrdResult.exitCode === 0) { // Parse the PRD
testResults.steps.parsePrd = true; const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], {
logger.success('PRD parsed successfully'); cwd: testDir,
timeout: 180000
// Extract task count from output });
const taskCountMatch = parsePrdResult.stdout.match(/(\d+) tasks? created/i);
if (taskCountMatch) {
logger.info(`Created ${taskCountMatch[1]} tasks from PRD`);
}
} else {
throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`);
}
// Step 6: Run complexity analysis if (parsePrdResult.exitCode === 0) {
logger.step('Running complexity analysis on parsed tasks'); testResults.steps.parsePrd = true;
// Ensure reports directory exists logger.success('PRD parsed successfully');
const reportsDir = join(testDir, '.taskmaster/reports');
if (!existsSync(reportsDir)) {
mkdirSync(reportsDir, { recursive: true });
}
const analyzeResult = await helpers.taskMaster('analyze-complexity', ['--research', '--output', '.taskmaster/reports/task-complexity-report.json'], {
cwd: testDir,
timeout: 240000
});
if (analyzeResult.exitCode === 0) { // Extract task count from output
testResults.steps.analyzeComplexity = true; const taskCountMatch =
logger.success('Complexity analysis completed'); parsePrdResult.stdout.match(/(\d+) tasks? created/i);
if (taskCountMatch) {
// Extract complexity information from output logger.info(`Created ${taskCountMatch[1]} tasks from PRD`);
const complexityMatch = analyzeResult.stdout.match(/Total Complexity Score: ([\d.]+)/); }
if (complexityMatch) { } else {
logger.info(`Total complexity score: ${complexityMatch[1]}`); throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`);
} }
} else {
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`);
}
// Step 7: Generate complexity report // Step 6: Run complexity analysis
logger.step('Generating complexity report'); logger.step('Running complexity analysis on parsed tasks');
const reportResult = await helpers.taskMaster('complexity-report', [], { // Ensure reports directory exists
cwd: testDir, const reportsDir = join(testDir, '.taskmaster/reports');
timeout: 60000 if (!existsSync(reportsDir)) {
}); mkdirSync(reportsDir, { recursive: true });
}
const analyzeResult = await helpers.taskMaster(
'analyze-complexity',
[
'--research',
'--output',
'.taskmaster/reports/task-complexity-report.json'
],
{
cwd: testDir,
timeout: 240000
}
);
if (reportResult.exitCode === 0) { if (analyzeResult.exitCode === 0) {
testResults.steps.generateReport = true; testResults.steps.analyzeComplexity = true;
logger.success('Complexity report generated'); logger.success('Complexity analysis completed');
// Check if complexity report file was created (not needed since complexity-report reads from the standard location)
const reportPath = join(testDir, '.taskmaster/reports/task-complexity-report.json');
if (helpers.fileExists(reportPath)) {
testResults.complexityReport = helpers.readJson(reportPath);
logger.info('Complexity report saved to task-complexity-report.json');
// Log summary if available
if (testResults.complexityReport && testResults.complexityReport.summary) {
const summary = testResults.complexityReport.summary;
logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`);
logger.info(`Average complexity: ${summary.averageComplexity || 0}`);
}
}
} else {
logger.warning(`Complexity report generation had issues: ${reportResult.stderr}`);
// Don't fail the test for report generation issues
testResults.steps.generateReport = true;
}
// Verify tasks.json was created // Extract complexity information from output
const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json'); const complexityMatch = analyzeResult.stdout.match(
if (helpers.fileExists(tasksJsonPath)) { /Total Complexity Score: ([\d.]+)/
const taskCount = helpers.getTaskCount(tasksJsonPath); );
logger.info(`Verified tasks.json exists with ${taskCount} tasks`); if (complexityMatch) {
} else { logger.info(`Total complexity score: ${complexityMatch[1]}`);
throw new Error('tasks.json was not created'); }
} } else {
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`);
}
// All steps completed successfully // Step 7: Generate complexity report
testResults.status = 'success'; logger.step('Generating complexity report');
logger.success('Setup test completed successfully'); const reportResult = await helpers.taskMaster('complexity-report', [], {
cwd: testDir,
timeout: 60000
});
} catch (error) { if (reportResult.exitCode === 0) {
testResults.status = 'failed'; testResults.steps.generateReport = true;
testResults.errors.push(error.message); logger.success('Complexity report generated');
logger.error(`Setup test failed: ${error.message}`);
// Log which steps completed
logger.info('Completed steps:');
Object.entries(testResults.steps).forEach(([step, completed]) => {
if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
// Flush logs before returning // Check if complexity report file was created (not needed since complexity-report reads from the standard location)
logger.flush(); const reportPath = join(
testDir,
'.taskmaster/reports/task-complexity-report.json'
);
if (helpers.fileExists(reportPath)) {
testResults.complexityReport = helpers.readJson(reportPath);
logger.info('Complexity report saved to task-complexity-report.json');
return testResults; // Log summary if available
if (
testResults.complexityReport &&
testResults.complexityReport.summary
) {
const summary = testResults.complexityReport.summary;
logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`);
logger.info(`Average complexity: ${summary.averageComplexity || 0}`);
}
}
} else {
logger.warning(
`Complexity report generation had issues: ${reportResult.stderr}`
);
// Don't fail the test for report generation issues
testResults.steps.generateReport = true;
}
// Verify tasks.json was created
const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json');
if (helpers.fileExists(tasksJsonPath)) {
const taskCount = helpers.getTaskCount(tasksJsonPath);
logger.info(`Verified tasks.json exists with ${taskCount} tasks`);
} else {
throw new Error('tasks.json was not created');
}
// All steps completed successfully
testResults.status = 'success';
logger.success('Setup test completed successfully');
} catch (error) {
testResults.status = 'failed';
testResults.errors.push(error.message);
logger.error(`Setup test failed: ${error.message}`);
// Log which steps completed
logger.info('Completed steps:');
Object.entries(testResults.steps).forEach(([step, completed]) => {
if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
// Flush logs before returning
logger.flush();
return testResults;
} }
// Export default for direct execution // Export default for direct execution
export default runSetupTest; export default runSetupTest;

403
tests/e2e/utils/error-handler.js
View File

@@ -3,234 +3,245 @@ import { join } from 'path';
import chalk from 'chalk'; import chalk from 'chalk';
export class ErrorHandler { export class ErrorHandler {
constructor(logger) { constructor(logger) {
this.logger = logger; this.logger = logger;
this.errors = []; this.errors = [];
this.warnings = []; this.warnings = [];
} }
/** /**
* Handle and categorize errors * Handle and categorize errors
*/ */
handleError(error, context = {}) { handleError(error, context = {}) {
const errorInfo = { const errorInfo = {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
message: error.message || 'Unknown error', message: error.message || 'Unknown error',
stack: error.stack, stack: error.stack,
context, context,
type: this.categorizeError(error) type: this.categorizeError(error)
}; };
this.errors.push(errorInfo); this.errors.push(errorInfo);
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`); this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
if (context.critical) { if (context.critical) {
throw error; throw error;
} }
return errorInfo; return errorInfo;
} }
/** /**
* Add a warning * Add a warning
*/ */
addWarning(message, context = {}) { addWarning(message, context = {}) {
const warning = { const warning = {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
message, message,
context context
}; };
this.warnings.push(warning); this.warnings.push(warning);
this.logger.warning(message); this.logger.warning(message);
} }
/** /**
* Categorize error types * Categorize error types
*/ */
categorizeError(error) { categorizeError(error) {
const message = error.message.toLowerCase(); const message = error.message.toLowerCase();
if (message.includes('command not found') || message.includes('not found')) { if (
return 'DEPENDENCY_ERROR'; message.includes('command not found') ||
} message.includes('not found')
if (message.includes('permission') || message.includes('access denied')) { ) {
return 'PERMISSION_ERROR'; return 'DEPENDENCY_ERROR';
} }
if (message.includes('timeout')) { if (message.includes('permission') || message.includes('access denied')) {
return 'TIMEOUT_ERROR'; return 'PERMISSION_ERROR';
} }
if (message.includes('api') || message.includes('rate limit')) { if (message.includes('timeout')) {
return 'API_ERROR'; return 'TIMEOUT_ERROR';
} }
if (message.includes('json') || message.includes('parse')) { if (message.includes('api') || message.includes('rate limit')) {
return 'PARSE_ERROR'; return 'API_ERROR';
} }
if (message.includes('file') || message.includes('directory')) { if (message.includes('json') || message.includes('parse')) {
return 'FILE_ERROR'; return 'PARSE_ERROR';
} }
if (message.includes('file') || message.includes('directory')) {
return 'FILE_ERROR';
}
return 'GENERAL_ERROR'; return 'GENERAL_ERROR';
} }
/** /**
* Get error summary * Get error summary
*/ */
getSummary() { getSummary() {
const errorsByType = {}; const errorsByType = {};
this.errors.forEach(error => {
if (!errorsByType[error.type]) {
errorsByType[error.type] = [];
}
errorsByType[error.type].push(error);
});
return { this.errors.forEach((error) => {
totalErrors: this.errors.length, if (!errorsByType[error.type]) {
totalWarnings: this.warnings.length, errorsByType[error.type] = [];
errorsByType, }
criticalErrors: this.errors.filter(e => e.context.critical), errorsByType[error.type].push(error);
recentErrors: this.errors.slice(-5) });
};
}
/** return {
* Generate error report totalErrors: this.errors.length,
*/ totalWarnings: this.warnings.length,
generateReport(outputPath) { errorsByType,
const summary = this.getSummary(); criticalErrors: this.errors.filter((e) => e.context.critical),
const report = { recentErrors: this.errors.slice(-5)
generatedAt: new Date().toISOString(), };
summary: { }
totalErrors: summary.totalErrors,
totalWarnings: summary.totalWarnings,
errorTypes: Object.keys(summary.errorsByType)
},
errors: this.errors,
warnings: this.warnings,
recommendations: this.generateRecommendations(summary)
};
writeFileSync(outputPath, JSON.stringify(report, null, 2)); /**
return report; * Generate error report
} */
generateReport(outputPath) {
const summary = this.getSummary();
const report = {
generatedAt: new Date().toISOString(),
summary: {
totalErrors: summary.totalErrors,
totalWarnings: summary.totalWarnings,
errorTypes: Object.keys(summary.errorsByType)
},
errors: this.errors,
warnings: this.warnings,
recommendations: this.generateRecommendations(summary)
};
/** writeFileSync(outputPath, JSON.stringify(report, null, 2));
* Generate recommendations based on errors return report;
*/ }
generateRecommendations(summary) {
const recommendations = [];
if (summary.errorsByType.DEPENDENCY_ERROR) { /**
recommendations.push({ * Generate recommendations based on errors
type: 'DEPENDENCY', */
message: 'Install missing dependencies using npm install or check PATH', generateRecommendations(summary) {
errors: summary.errorsByType.DEPENDENCY_ERROR.length const recommendations = [];
});
}
if (summary.errorsByType.PERMISSION_ERROR) { if (summary.errorsByType.DEPENDENCY_ERROR) {
recommendations.push({ recommendations.push({
type: 'PERMISSION', type: 'DEPENDENCY',
message: 'Check file permissions or run with appropriate privileges', message: 'Install missing dependencies using npm install or check PATH',
errors: summary.errorsByType.PERMISSION_ERROR.length errors: summary.errorsByType.DEPENDENCY_ERROR.length
}); });
} }
if (summary.errorsByType.API_ERROR) { if (summary.errorsByType.PERMISSION_ERROR) {
recommendations.push({ recommendations.push({
type: 'API', type: 'PERMISSION',
message: 'Check API keys, rate limits, or network connectivity', message: 'Check file permissions or run with appropriate privileges',
errors: summary.errorsByType.API_ERROR.length errors: summary.errorsByType.PERMISSION_ERROR.length
}); });
} }
if (summary.errorsByType.TIMEOUT_ERROR) { if (summary.errorsByType.API_ERROR) {
recommendations.push({ recommendations.push({
type: 'TIMEOUT', type: 'API',
message: 'Consider increasing timeout values or optimizing slow operations', message: 'Check API keys, rate limits, or network connectivity',
errors: summary.errorsByType.TIMEOUT_ERROR.length errors: summary.errorsByType.API_ERROR.length
}); });
} }
return recommendations; if (summary.errorsByType.TIMEOUT_ERROR) {
} recommendations.push({
type: 'TIMEOUT',
message:
'Consider increasing timeout values or optimizing slow operations',
errors: summary.errorsByType.TIMEOUT_ERROR.length
});
}
/** return recommendations;
* Display error summary in console }
*/
displaySummary() {
const summary = this.getSummary();
if (summary.totalErrors === 0 && summary.totalWarnings === 0) { /**
console.log(chalk.green('✅ No errors or warnings detected')); * Display error summary in console
return; */
} displaySummary() {
const summary = this.getSummary();
console.log(chalk.red.bold(`\n🚨 Error Summary:`)); if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`)); console.log(chalk.green('✅ No errors or warnings detected'));
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`)); return;
}
if (summary.totalErrors > 0) { console.log(chalk.red.bold(`\n🚨 Error Summary:`));
console.log(chalk.red.bold('\n Error Types:')); console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
Object.entries(summary.errorsByType).forEach(([type, errors]) => { console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
console.log(chalk.red(` - ${type}: ${errors.length}`));
});
if (summary.criticalErrors.length > 0) { if (summary.totalErrors > 0) {
console.log(chalk.red.bold(`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`)); console.log(chalk.red.bold('\n Error Types:'));
summary.criticalErrors.forEach(error => { Object.entries(summary.errorsByType).forEach(([type, errors]) => {
console.log(chalk.red(` - ${error.message}`)); console.log(chalk.red(` - ${type}: ${errors.length}`));
}); });
}
}
const recommendations = this.generateRecommendations(summary); if (summary.criticalErrors.length > 0) {
if (recommendations.length > 0) { console.log(
console.log(chalk.yellow.bold('\n💡 Recommendations:')); chalk.red.bold(
recommendations.forEach(rec => { `\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`
console.log(chalk.yellow(` - ${rec.message}`)); )
}); );
} summary.criticalErrors.forEach((error) => {
} console.log(chalk.red(` - ${error.message}`));
});
}
}
/** const recommendations = this.generateRecommendations(summary);
* Clear all errors and warnings if (recommendations.length > 0) {
*/ console.log(chalk.yellow.bold('\n💡 Recommendations:'));
clear() { recommendations.forEach((rec) => {
this.errors = []; console.log(chalk.yellow(` - ${rec.message}`));
this.warnings = []; });
} }
}
/**
* Clear all errors and warnings
*/
clear() {
this.errors = [];
this.warnings = [];
}
} }
/** /**
* Global error handler for uncaught exceptions * Global error handler for uncaught exceptions
*/ */
export function setupGlobalErrorHandlers(errorHandler, logger) { export function setupGlobalErrorHandlers(errorHandler, logger) {
process.on('uncaughtException', (error) => { process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`); logger.error(`Uncaught Exception: ${error.message}`);
errorHandler.handleError(error, { critical: true, source: 'uncaughtException' }); errorHandler.handleError(error, {
process.exit(1); critical: true,
}); source: 'uncaughtException'
});
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => { process.on('unhandledRejection', (reason, promise) => {
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`); logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
errorHandler.handleError(new Error(String(reason)), { errorHandler.handleError(new Error(String(reason)), {
critical: false, critical: false,
source: 'unhandledRejection' source: 'unhandledRejection'
}); });
}); });
process.on('SIGINT', () => { process.on('SIGINT', () => {
logger.info('\nReceived SIGINT, shutting down gracefully...'); logger.info('\nReceived SIGINT, shutting down gracefully...');
errorHandler.displaySummary(); errorHandler.displaySummary();
process.exit(130); process.exit(130);
}); });
process.on('SIGTERM', () => { process.on('SIGTERM', () => {
logger.info('\nReceived SIGTERM, shutting down...'); logger.info('\nReceived SIGTERM, shutting down...');
errorHandler.displaySummary(); errorHandler.displaySummary();
process.exit(143); process.exit(143);
}); });
} }

223
tests/e2e/utils/llm-analyzer.js
View File

@@ -2,56 +2,58 @@ import { readFileSync } from 'fs';
import fetch from 'node-fetch'; import fetch from 'node-fetch';
export class LLMAnalyzer { export class LLMAnalyzer {
constructor(config, logger) { constructor(config, logger) {
this.config = config; this.config = config;
this.logger = logger; this.logger = logger;
this.apiKey = process.env.ANTHROPIC_API_KEY; this.apiKey = process.env.ANTHROPIC_API_KEY;
this.apiEndpoint = 'https://api.anthropic.com/v1/messages'; this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
} }
async analyzeLog(logFile, providerSummaryFile = null) { async analyzeLog(logFile, providerSummaryFile = null) {
if (!this.config.llmAnalysis.enabled) { if (!this.config.llmAnalysis.enabled) {
this.logger.info('LLM analysis is disabled in configuration'); this.logger.info('LLM analysis is disabled in configuration');
return null; return null;
} }
if (!this.apiKey) { if (!this.apiKey) {
this.logger.error('ANTHROPIC_API_KEY not found in environment'); this.logger.error('ANTHROPIC_API_KEY not found in environment');
return null; return null;
} }
try { try {
const logContent = readFileSync(logFile, 'utf8'); const logContent = readFileSync(logFile, 'utf8');
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile); const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
const response = await this.callLLM(prompt); const response = await this.callLLM(prompt);
const analysis = this.parseResponse(response); const analysis = this.parseResponse(response);
// Calculate and log cost
if (response.usage) {
const cost = this.calculateCost(response.usage);
this.logger.addCost(cost);
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
}
return analysis; // Calculate and log cost
} catch (error) { if (response.usage) {
this.logger.error(`LLM analysis failed: ${error.message}`); const cost = this.calculateCost(response.usage);
return null; this.logger.addCost(cost);
} this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
} }
buildAnalysisPrompt(logContent, providerSummaryFile) { return analysis;
let providerSummary = ''; } catch (error) {
if (providerSummaryFile) { this.logger.error(`LLM analysis failed: ${error.message}`);
try { return null;
providerSummary = readFileSync(providerSummaryFile, 'utf8'); }
} catch (error) { }
this.logger.warning(`Could not read provider summary file: ${error.message}`);
}
}
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially. buildAnalysisPrompt(logContent, providerSummaryFile) {
let providerSummary = '';
if (providerSummaryFile) {
try {
providerSummary = readFileSync(providerSummaryFile, 'utf8');
} catch (error) {
this.logger.warning(
`Could not read provider summary file: ${error.message}`
);
}
}
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
Your goal is to: Your goal is to:
1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files). 1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files).
@@ -88,81 +90,82 @@ Return your analysis **strictly** in the following JSON format. Do not include a
Here is the main log content: Here is the main log content:
${logContent}`; ${logContent}`;
} }
async callLLM(prompt) { async callLLM(prompt) {
const payload = { const payload = {
model: this.config.llmAnalysis.model, model: this.config.llmAnalysis.model,
max_tokens: this.config.llmAnalysis.maxTokens, max_tokens: this.config.llmAnalysis.maxTokens,
messages: [ messages: [{ role: 'user', content: prompt }]
{ role: 'user', content: prompt } };
]
};
const response = await fetch(this.apiEndpoint, { const response = await fetch(this.apiEndpoint, {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'x-api-key': this.apiKey, 'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01' 'anthropic-version': '2023-06-01'
}, },
body: JSON.stringify(payload) body: JSON.stringify(payload)
}); });
if (!response.ok) { if (!response.ok) {
const error = await response.text(); const error = await response.text();
throw new Error(`LLM API call failed: ${response.status} - ${error}`); throw new Error(`LLM API call failed: ${response.status} - ${error}`);
} }
return response.json(); return response.json();
} }
parseResponse(response) { parseResponse(response) {
try { try {
const content = response.content[0].text; const content = response.content[0].text;
const jsonStart = content.indexOf('{'); const jsonStart = content.indexOf('{');
const jsonEnd = content.lastIndexOf('}'); const jsonEnd = content.lastIndexOf('}');
if (jsonStart === -1 || jsonEnd === -1) {
throw new Error('No JSON found in response');
}
const jsonString = content.substring(jsonStart, jsonEnd + 1); if (jsonStart === -1 || jsonEnd === -1) {
return JSON.parse(jsonString); throw new Error('No JSON found in response');
} catch (error) { }
this.logger.error(`Failed to parse LLM response: ${error.message}`);
return null;
}
}
calculateCost(usage) { const jsonString = content.substring(jsonStart, jsonEnd + 1);
const modelCosts = { return JSON.parse(jsonString);
'claude-3-7-sonnet-20250219': { } catch (error) {
input: 3.00, // per 1M tokens this.logger.error(`Failed to parse LLM response: ${error.message}`);
output: 15.00 // per 1M tokens return null;
} }
}; }
const costs = modelCosts[this.config.llmAnalysis.model] || { input: 0, output: 0 }; calculateCost(usage) {
const inputCost = (usage.input_tokens / 1000000) * costs.input; const modelCosts = {
const outputCost = (usage.output_tokens / 1000000) * costs.output; 'claude-3-7-sonnet-20250219': {
input: 3.0, // per 1M tokens
return inputCost + outputCost; output: 15.0 // per 1M tokens
} }
};
formatReport(analysis) { const costs = modelCosts[this.config.llmAnalysis.model] || {
if (!analysis) return null; input: 0,
output: 0
};
const inputCost = (usage.input_tokens / 1000000) * costs.input;
const outputCost = (usage.output_tokens / 1000000) * costs.output;
const report = { return inputCost + outputCost;
title: 'TASKMASTER E2E Test Analysis Report', }
timestamp: new Date().toISOString(),
status: analysis.overall_status,
summary: analysis.llm_summary_points,
verifiedSteps: analysis.verified_steps,
providerComparison: analysis.provider_add_task_comparison,
issues: analysis.detected_issues
};
return report; formatReport(analysis) {
} if (!analysis) return null;
}
const report = {
title: 'TASKMASTER E2E Test Analysis Report',
timestamp: new Date().toISOString(),
status: analysis.overall_status,
summary: analysis.llm_summary_points,
verifiedSteps: analysis.verified_steps,
providerComparison: analysis.provider_add_task_comparison,
issues: analysis.detected_issues
};
return report;
}
}

217
tests/e2e/utils/logger.js
View File

@@ -3,122 +3,129 @@ import { join } from 'path';
import chalk from 'chalk'; import chalk from 'chalk';
export class TestLogger { export class TestLogger {
constructor(logDir, testRunId) { constructor(logDir, testRunId) {
this.logDir = logDir; this.logDir = logDir;
this.testRunId = testRunId; this.testRunId = testRunId;
this.startTime = Date.now(); this.startTime = Date.now();
this.stepCount = 0; this.stepCount = 0;
this.logFile = join(logDir, `e2e_run_${testRunId}.log`); this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
this.logBuffer = []; this.logBuffer = [];
this.totalCost = 0; this.totalCost = 0;
// Ensure log directory exists // Ensure log directory exists
if (!existsSync(logDir)) { if (!existsSync(logDir)) {
mkdirSync(logDir, { recursive: true }); mkdirSync(logDir, { recursive: true });
} }
} }
formatDuration(milliseconds) { formatDuration(milliseconds) {
const totalSeconds = Math.floor(milliseconds / 1000); const totalSeconds = Math.floor(milliseconds / 1000);
const minutes = Math.floor(totalSeconds / 60); const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60; const seconds = totalSeconds % 60;
return `${minutes}m${seconds.toString().padStart(2, '0')}s`; return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
} }
getElapsedTime() { getElapsedTime() {
return this.formatDuration(Date.now() - this.startTime); return this.formatDuration(Date.now() - this.startTime);
} }
formatLogEntry(level, message) { formatLogEntry(level, message) {
const timestamp = new Date().toISOString(); const timestamp = new Date().toISOString();
const elapsed = this.getElapsedTime(); const elapsed = this.getElapsedTime();
return `[${level}] [${elapsed}] ${timestamp} ${message}`; return `[${level}] [${elapsed}] ${timestamp} ${message}`;
} }
log(level, message, options = {}) { log(level, message, options = {}) {
const formattedMessage = this.formatLogEntry(level, message); const formattedMessage = this.formatLogEntry(level, message);
// Add to buffer
this.logBuffer.push(formattedMessage);
// Console output with colors
let coloredMessage = formattedMessage;
switch (level) {
case 'INFO':
coloredMessage = chalk.blue(formattedMessage);
break;
case 'SUCCESS':
coloredMessage = chalk.green(formattedMessage);
break;
case 'ERROR':
coloredMessage = chalk.red(formattedMessage);
break;
case 'WARNING':
coloredMessage = chalk.yellow(formattedMessage);
break;
}
console.log(coloredMessage);
// Write to file if immediate flush requested
if (options.flush) {
this.flush();
}
}
info(message) { // Add to buffer
this.log('INFO', message); this.logBuffer.push(formattedMessage);
}
success(message) { // Console output with colors
this.log('SUCCESS', message); let coloredMessage = formattedMessage;
} switch (level) {
case 'INFO':
coloredMessage = chalk.blue(formattedMessage);
break;
case 'SUCCESS':
coloredMessage = chalk.green(formattedMessage);
break;
case 'ERROR':
coloredMessage = chalk.red(formattedMessage);
break;
case 'WARNING':
coloredMessage = chalk.yellow(formattedMessage);
break;
}
error(message) { console.log(coloredMessage);
this.log('ERROR', message);
}
warning(message) { // Write to file if immediate flush requested
this.log('WARNING', message); if (options.flush) {
} this.flush();
}
}
step(message) { info(message) {
this.stepCount++; this.log('INFO', message);
const separator = '='.repeat(45); }
this.log('STEP', `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`);
}
addCost(cost) { success(message) {
if (typeof cost === 'number' && !isNaN(cost)) { this.log('SUCCESS', message);
this.totalCost += cost; }
}
}
extractAndAddCost(output) { error(message) {
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g; this.log('ERROR', message);
let match; }
while ((match = costRegex.exec(output)) !== null) {
const cost = parseFloat(match[1]);
this.addCost(cost);
}
}
flush() { warning(message) {
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8'); this.log('WARNING', message);
} }
getSummary() { step(message) {
const duration = this.formatDuration(Date.now() - this.startTime); this.stepCount++;
const successCount = this.logBuffer.filter(line => line.includes('[SUCCESS]')).length; const separator = '='.repeat(45);
const errorCount = this.logBuffer.filter(line => line.includes('[ERROR]')).length; this.log(
'STEP',
return { `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`
duration, );
totalSteps: this.stepCount, }
successCount,
errorCount, addCost(cost) {
totalCost: this.totalCost.toFixed(6), if (typeof cost === 'number' && !isNaN(cost)) {
logFile: this.logFile this.totalCost += cost;
}; }
} }
}
extractAndAddCost(output) {
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
let match;
while ((match = costRegex.exec(output)) !== null) {
const cost = parseFloat(match[1]);
this.addCost(cost);
}
}
flush() {
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
}
getSummary() {
const duration = this.formatDuration(Date.now() - this.startTime);
const successCount = this.logBuffer.filter((line) =>
line.includes('[SUCCESS]')
).length;
const errorCount = this.logBuffer.filter((line) =>
line.includes('[ERROR]')
).length;
return {
duration,
totalSteps: this.stepCount,
successCount,
errorCount,
totalCost: this.totalCost.toFixed(6),
logFile: this.logFile
};
}
}

337
tests/e2e/utils/test-helpers.js
View File

@@ -3,185 +3,190 @@ import { readFileSync, existsSync, copyFileSync } from 'fs';
import { join } from 'path'; import { join } from 'path';
export class TestHelpers { export class TestHelpers {
constructor(logger) { constructor(logger) {
this.logger = logger; this.logger = logger;
} }
/** /**
* Execute a command and return output * Execute a command and return output
* @param {string} command - Command to execute * @param {string} command - Command to execute
* @param {string[]} args - Command arguments * @param {string[]} args - Command arguments
* @param {Object} options - Execution options * @param {Object} options - Execution options
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>} * @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
*/ */
async executeCommand(command, args = [], options = {}) { async executeCommand(command, args = [], options = {}) {
return new Promise((resolve) => { return new Promise((resolve) => {
const spawnOptions = { const spawnOptions = {
cwd: options.cwd || process.cwd(), cwd: options.cwd || process.cwd(),
env: { ...process.env, ...options.env }, env: { ...process.env, ...options.env },
shell: true shell: true
}; };
// When using shell: true, pass the full command as a single string // When using shell: true, pass the full command as a single string
const fullCommand = args.length > 0 ? `${command} ${args.join(' ')}` : command; const fullCommand =
const child = spawn(fullCommand, [], spawnOptions); args.length > 0 ? `${command} ${args.join(' ')}` : command;
let stdout = ''; const child = spawn(fullCommand, [], spawnOptions);
let stderr = ''; let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => { child.stdout.on('data', (data) => {
stdout += data.toString(); stdout += data.toString();
}); });
child.stderr.on('data', (data) => { child.stderr.on('data', (data) => {
stderr += data.toString(); stderr += data.toString();
}); });
child.on('close', (exitCode) => { child.on('close', (exitCode) => {
const output = stdout + stderr; const output = stdout + stderr;
// Extract and log costs
this.logger.extractAndAddCost(output);
resolve({ stdout, stderr, exitCode });
});
// Handle timeout // Extract and log costs
if (options.timeout) { this.logger.extractAndAddCost(output);
setTimeout(() => {
child.kill('SIGTERM');
}, options.timeout);
}
});
}
/** resolve({ stdout, stderr, exitCode });
* Execute task-master command });
* @param {string} subcommand - Task-master subcommand
* @param {string[]} args - Command arguments
* @param {Object} options - Execution options
*/
async taskMaster(subcommand, args = [], options = {}) {
const fullArgs = [subcommand, ...args];
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
const result = await this.executeCommand('task-master', fullArgs, options);
if (result.exitCode !== 0 && !options.allowFailure) {
this.logger.error(`Command failed with exit code ${result.exitCode}`);
this.logger.error(`stderr: ${result.stderr}`);
}
return result;
}
/** // Handle timeout
* Check if a file exists if (options.timeout) {
*/ setTimeout(() => {
fileExists(filePath) { child.kill('SIGTERM');
return existsSync(filePath); }, options.timeout);
} }
});
}
/** /**
* Read JSON file * Execute task-master command
*/ * @param {string} subcommand - Task-master subcommand
readJson(filePath) { * @param {string[]} args - Command arguments
try { * @param {Object} options - Execution options
const content = readFileSync(filePath, 'utf8'); */
return JSON.parse(content); async taskMaster(subcommand, args = [], options = {}) {
} catch (error) { const fullArgs = [subcommand, ...args];
this.logger.error(`Failed to read JSON file ${filePath}: ${error.message}`); this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
return null;
}
}
/** const result = await this.executeCommand('task-master', fullArgs, options);
* Copy file
*/
copyFile(source, destination) {
try {
copyFileSync(source, destination);
return true;
} catch (error) {
this.logger.error(`Failed to copy file from ${source} to ${destination}: ${error.message}`);
return false;
}
}
/** if (result.exitCode !== 0 && !options.allowFailure) {
* Wait for a specified duration this.logger.error(`Command failed with exit code ${result.exitCode}`);
*/ this.logger.error(`stderr: ${result.stderr}`);
async wait(milliseconds) { }
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
/** return result;
* Verify task exists in tasks.json }
*/
verifyTaskExists(tasksFile, taskId, tagName = 'master') {
const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return false;
return tasks[tagName].tasks.some(task => task.id === taskId);
}
/** /**
* Get task count for a tag * Check if a file exists
*/ */
getTaskCount(tasksFile, tagName = 'master') { fileExists(filePath) {
const tasks = this.readJson(tasksFile); return existsSync(filePath);
if (!tasks || !tasks[tagName]) return 0; }
return tasks[tagName].tasks.length;
}
/** /**
* Extract task ID from command output * Read JSON file
*/ */
extractTaskId(output) { readJson(filePath) {
const patterns = [ try {
/✓ Added new task #(\d+(?:\.\d+)?)/, const content = readFileSync(filePath, 'utf8');
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/, return JSON.parse(content);
/Task (\d+(?:\.\d+)?) Created Successfully/ } catch (error) {
]; this.logger.error(
`Failed to read JSON file ${filePath}: ${error.message}`
for (const pattern of patterns) { );
const match = output.match(pattern); return null;
if (match) { }
return match[1]; }
}
}
return null;
}
/** /**
* Run multiple async operations in parallel * Copy file
*/ */
async runParallel(operations) { copyFile(source, destination) {
return Promise.all(operations); try {
} copyFileSync(source, destination);
return true;
} catch (error) {
this.logger.error(
`Failed to copy file from ${source} to ${destination}: ${error.message}`
);
return false;
}
}
/** /**
* Run operations with concurrency limit * Wait for a specified duration
*/ */
async runWithConcurrency(operations, limit = 3) { async wait(milliseconds) {
const results = []; return new Promise((resolve) => setTimeout(resolve, milliseconds));
const executing = []; }
for (const operation of operations) { /**
const promise = operation().then(result => { * Verify task exists in tasks.json
executing.splice(executing.indexOf(promise), 1); */
return result; verifyTaskExists(tasksFile, taskId, tagName = 'master') {
}); const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return false;
results.push(promise);
executing.push(promise); return tasks[tagName].tasks.some((task) => task.id === taskId);
}
if (executing.length >= limit) {
await Promise.race(executing); /**
} * Get task count for a tag
} */
getTaskCount(tasksFile, tagName = 'master') {
return Promise.all(results); const tasks = this.readJson(tasksFile);
} if (!tasks || !tasks[tagName]) return 0;
}
return tasks[tagName].tasks.length;
}
/**
* Extract task ID from command output
*/
extractTaskId(output) {
const patterns = [
/✓ Added new task #(\d+(?:\.\d+)?)/,
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
/Task (\d+(?:\.\d+)?) Created Successfully/
];
for (const pattern of patterns) {
const match = output.match(pattern);
if (match) {
return match[1];
}
}
return null;
}
/**
* Run multiple async operations in parallel
*/
async runParallel(operations) {
return Promise.all(operations);
}
/**
* Run operations with concurrency limit
*/
async runWithConcurrency(operations, limit = 3) {
const results = [];
const executing = [];
for (const operation of operations) {
const promise = operation().then((result) => {
executing.splice(executing.indexOf(promise), 1);
return result;
});
results.push(promise);
executing.push(promise);
if (executing.length >= limit) {
await Promise.race(executing);
}
}
return Promise.all(results);
}
}