ros/claude-task-master
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: run format

Browse Source
This commit is contained in:
Ralph Khreish
2025-07-08 09:40:30 +03:00
parent 395693af24
commit d4208f372a
12 changed files with 2099 additions and 1776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
tests/e2e/config/test-config.js
View File

@@ -10,56 +10,63 @@ const projectRoot = join(__dirname, '../../..');
dotenvConfig({ path: join(projectRoot, '.env') });
export const testConfig = {
// Paths
paths: {
projectRoot,
sourceDir: projectRoot,
baseTestDir: join(projectRoot, 'tests/e2e/_runs'),
logDir: join(projectRoot, 'tests/e2e/log'),
samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'),
mainEnvFile: join(projectRoot, '.env'),
supportedModelsFile: join(projectRoot, 'scripts/modules/supported-models.json')
},
// Paths
paths: {
projectRoot,
sourceDir: projectRoot,
baseTestDir: join(projectRoot, 'tests/e2e/_runs'),
logDir: join(projectRoot, 'tests/e2e/log'),
samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'),
mainEnvFile: join(projectRoot, '.env'),
supportedModelsFile: join(
projectRoot,
'scripts/modules/supported-models.json'
)
},
// Test settings
settings: {
runVerificationTest: true,
parallelTestGroups: 4, // Number of parallel test groups
timeout: 600000, // 10 minutes default timeout
retryAttempts: 2
},
// Test settings
settings: {
runVerificationTest: true,
parallelTestGroups: 4, // Number of parallel test groups
timeout: 600000, // 10 minutes default timeout
retryAttempts: 2
},
// Provider test configuration
providers: [
{ name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] },
{ name: 'openai', model: 'gpt-4o', flags: [] },
{ name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] },
{ name: 'perplexity', model: 'sonar-pro', flags: [] },
{ name: 'xai', model: 'grok-3', flags: [] },
{ name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] }
],
// Provider test configuration
providers: [
{ name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] },
{ name: 'openai', model: 'gpt-4o', flags: [] },
{ name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] },
{ name: 'perplexity', model: 'sonar-pro', flags: [] },
{ name: 'xai', model: 'grok-3', flags: [] },
{ name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] }
],
// Test prompts
prompts: {
addTask: 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.',
updateTask: 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.',
updateFromTask: 'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.',
updateSubtask: 'Implementation note: Remember to handle potential API errors and display a user-friendly message.'
},
// Test prompts
prompts: {
addTask:
'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.',
updateTask:
'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.',
updateFromTask:
'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.',
updateSubtask:
'Implementation note: Remember to handle potential API errors and display a user-friendly message.'
},
// LLM Analysis settings
llmAnalysis: {
enabled: true,
model: 'claude-3-7-sonnet-20250219',
provider: 'anthropic',
maxTokens: 3072
}
// LLM Analysis settings
llmAnalysis: {
enabled: true,
model: 'claude-3-7-sonnet-20250219',
provider: 'anthropic',
maxTokens: 3072
}
};
// Export test groups for parallel execution
export const testGroups = {
setup: ['setup'],
core: ['core'],
providers: ['providers'],
advanced: ['advanced']
};
setup: ['setup'],
core: ['core'],
providers: ['providers'],
advanced: ['advanced']
};

7
tests/e2e/run-e2e-tests.js
View File

@@ -237,14 +237,15 @@ async function runTests(options) {
}
// Check if we need to run setup (either explicitly requested or needed for other tests)
const needsSetup = testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0);
const needsSetup =
testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0);
if (needsSetup) {
// Always run setup if we need a test directory
if (!testsToRun.setup) {
logger.info('No test directory available, running setup automatically');
}
logger.step('Running setup tests');
const setupRunner = new SequentialTestRunner(logger, helpers);
const setupResults = await setupRunner.runTests(['setup'], {});

370
tests/e2e/runners/parallel-runner.js
View File

@@ -7,205 +7,219 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
export class ParallelTestRunner extends EventEmitter {
constructor(logger) {
super();
this.logger = logger;
this.workers = [];
this.results = {};
}
constructor(logger) {
super();
this.logger = logger;
this.workers = [];
this.results = {};
}
/**
* Run test groups in parallel
* @param {Object} testGroups - Groups of tests to run
* @param {Object} sharedContext - Shared context for all tests
* @returns {Promise<Object>} Combined results from all test groups
*/
async runTestGroups(testGroups, sharedContext) {
const groupNames = Object.keys(testGroups);
const workerPromises = [];
/**
* Run test groups in parallel
* @param {Object} testGroups - Groups of tests to run
* @param {Object} sharedContext - Shared context for all tests
* @returns {Promise<Object>} Combined results from all test groups
*/
async runTestGroups(testGroups, sharedContext) {
const groupNames = Object.keys(testGroups);
const workerPromises = [];
this.logger.info(`Starting parallel execution of ${groupNames.length} test groups`);
this.logger.info(
`Starting parallel execution of ${groupNames.length} test groups`
);
for (const groupName of groupNames) {
const workerPromise = this.runTestGroup(groupName, testGroups[groupName], sharedContext);
workerPromises.push(workerPromise);
}
for (const groupName of groupNames) {
const workerPromise = this.runTestGroup(
groupName,
testGroups[groupName],
sharedContext
);
workerPromises.push(workerPromise);
}
// Wait for all workers to complete
const results = await Promise.allSettled(workerPromises);
// Wait for all workers to complete
const results = await Promise.allSettled(workerPromises);
// Process results
const combinedResults = {
overall: 'passed',
groups: {},
summary: {
totalGroups: groupNames.length,
passedGroups: 0,
failedGroups: 0,
errors: []
}
};
// Process results
const combinedResults = {
overall: 'passed',
groups: {},
summary: {
totalGroups: groupNames.length,
passedGroups: 0,
failedGroups: 0,
errors: []
}
};
results.forEach((result, index) => {
const groupName = groupNames[index];
if (result.status === 'fulfilled') {
combinedResults.groups[groupName] = result.value;
if (result.value.status === 'passed') {
combinedResults.summary.passedGroups++;
} else {
combinedResults.summary.failedGroups++;
combinedResults.overall = 'failed';
}
} else {
combinedResults.groups[groupName] = {
status: 'failed',
error: result.reason.message || 'Unknown error'
};
combinedResults.summary.failedGroups++;
combinedResults.summary.errors.push({
group: groupName,
error: result.reason.message
});
combinedResults.overall = 'failed';
}
});
results.forEach((result, index) => {
const groupName = groupNames[index];
return combinedResults;
}
if (result.status === 'fulfilled') {
combinedResults.groups[groupName] = result.value;
if (result.value.status === 'passed') {
combinedResults.summary.passedGroups++;
} else {
combinedResults.summary.failedGroups++;
combinedResults.overall = 'failed';
}
} else {
combinedResults.groups[groupName] = {
status: 'failed',
error: result.reason.message || 'Unknown error'
};
combinedResults.summary.failedGroups++;
combinedResults.summary.errors.push({
group: groupName,
error: result.reason.message
});
combinedResults.overall = 'failed';
}
});
/**
* Run a single test group in a worker thread
*/
async runTestGroup(groupName, testModules, sharedContext) {
return new Promise((resolve, reject) => {
const workerPath = join(__dirname, 'test-worker.js');
const worker = new Worker(workerPath, {
workerData: {
groupName,
testModules,
sharedContext,
logDir: this.logger.logDir,
testRunId: this.logger.testRunId
}
});
return combinedResults;
}
this.workers.push(worker);
/**
* Run a single test group in a worker thread
*/
async runTestGroup(groupName, testModules, sharedContext) {
return new Promise((resolve, reject) => {
const workerPath = join(__dirname, 'test-worker.js');
// Handle messages from worker
worker.on('message', (message) => {
if (message.type === 'log') {
const level = message.level.toLowerCase();
if (typeof this.logger[level] === 'function') {
this.logger[level](message.message);
} else {
// Fallback to info if the level doesn't exist
this.logger.info(message.message);
}
} else if (message.type === 'step') {
this.logger.step(message.message);
} else if (message.type === 'cost') {
this.logger.addCost(message.cost);
} else if (message.type === 'results') {
this.results[groupName] = message.results;
}
});
const worker = new Worker(workerPath, {
workerData: {
groupName,
testModules,
sharedContext,
logDir: this.logger.logDir,
testRunId: this.logger.testRunId
}
});
// Handle worker completion
worker.on('exit', (code) => {
this.workers = this.workers.filter(w => w !== worker);
if (code === 0) {
resolve(this.results[groupName] || { status: 'passed', group: groupName });
} else {
reject(new Error(`Worker for group ${groupName} exited with code ${code}`));
}
});
this.workers.push(worker);
// Handle worker errors
worker.on('error', (error) => {
this.workers = this.workers.filter(w => w !== worker);
reject(error);
});
// Handle messages from worker
worker.on('message', (message) => {
if (message.type === 'log') {
const level = message.level.toLowerCase();
if (typeof this.logger[level] === 'function') {
this.logger[level](message.message);
} else {
// Fallback to info if the level doesn't exist
this.logger.info(message.message);
}
} else if (message.type === 'step') {
this.logger.step(message.message);
} else if (message.type === 'cost') {
this.logger.addCost(message.cost);
} else if (message.type === 'results') {
this.results[groupName] = message.results;
}
});
});
}
// Handle worker completion
worker.on('exit', (code) => {
this.workers = this.workers.filter((w) => w !== worker);
/**
* Terminate all running workers
*/
async terminate() {
const terminationPromises = this.workers.map(worker =>
worker.terminate().catch(err =>
this.logger.warning(`Failed to terminate worker: ${err.message}`)
)
);
if (code === 0) {
resolve(
this.results[groupName] || { status: 'passed', group: groupName }
);
} else {
reject(
new Error(`Worker for group ${groupName} exited with code ${code}`)
);
}
});
await Promise.all(terminationPromises);
this.workers = [];
}
// Handle worker errors
worker.on('error', (error) => {
this.workers = this.workers.filter((w) => w !== worker);
reject(error);
});
});
}
/**
* Terminate all running workers
*/
async terminate() {
const terminationPromises = this.workers.map((worker) =>
worker
.terminate()
.catch((err) =>
this.logger.warning(`Failed to terminate worker: ${err.message}`)
)
);
await Promise.all(terminationPromises);
this.workers = [];
}
}
/**
* Sequential test runner for comparison or fallback
*/
export class SequentialTestRunner {
constructor(logger, helpers) {
this.logger = logger;
this.helpers = helpers;
}
constructor(logger, helpers) {
this.logger = logger;
this.helpers = helpers;
}
/**
* Run tests sequentially
*/
async runTests(testModules, context) {
const results = {
overall: 'passed',
tests: {},
summary: {
totalTests: testModules.length,
passedTests: 0,
failedTests: 0,
errors: []
}
};
/**
* Run tests sequentially
*/
async runTests(testModules, context) {
const results = {
overall: 'passed',
tests: {},
summary: {
totalTests: testModules.length,
passedTests: 0,
failedTests: 0,
errors: []
}
};
for (const testModule of testModules) {
try {
this.logger.step(`Running ${testModule} tests`);
// Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`);
const { default: testFn } = await import(testPath);
// Run the test
const testResults = await testFn(this.logger, this.helpers, context);
results.tests[testModule] = testResults;
if (testResults.status === 'passed') {
results.summary.passedTests++;
} else {
results.summary.failedTests++;
results.overall = 'failed';
}
} catch (error) {
this.logger.error(`Failed to run ${testModule}: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
for (const testModule of testModules) {
try {
this.logger.step(`Running ${testModule} tests`);
return results;
}
}
// Dynamic import of test module
const testPath = join(
dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test
const testResults = await testFn(this.logger, this.helpers, context);
results.tests[testModule] = testResults;
if (testResults.status === 'passed') {
results.summary.passedTests++;
} else {
results.summary.failedTests++;
results.overall = 'failed';
}
} catch (error) {
this.logger.error(`Failed to run ${testModule}: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message
};
results.summary.failedTests++;
results.summary.errors.push({
test: testModule,
error: error.message
});
results.overall = 'failed';
}
}
return results;
}
}

217
tests/e2e/runners/test-worker.js
View File

@@ -9,124 +9,127 @@ const __dirname = dirname(__filename);
// Worker logger that sends messages to parent
class WorkerLogger extends TestLogger {
constructor(logDir, testRunId, groupName) {
super(logDir, `${testRunId}_${groupName}`);
this.groupName = groupName;
}
constructor(logDir, testRunId, groupName) {
super(logDir, `${testRunId}_${groupName}`);
this.groupName = groupName;
}
log(level, message, options = {}) {
super.log(level, message, options);
// Send log to parent
parentPort.postMessage({
type: 'log',
level: level.toLowerCase(),
message: `[${this.groupName}] ${message}`
});
}
log(level, message, options = {}) {
super.log(level, message, options);
step(message) {
super.step(message);
parentPort.postMessage({
type: 'step',
message: `[${this.groupName}] ${message}`
});
}
// Send log to parent
parentPort.postMessage({
type: 'log',
level: level.toLowerCase(),
message: `[${this.groupName}] ${message}`
});
}
addCost(cost) {
super.addCost(cost);
parentPort.postMessage({
type: 'cost',
cost
});
}
step(message) {
super.step(message);
parentPort.postMessage({
type: 'step',
message: `[${this.groupName}] ${message}`
});
}
addCost(cost) {
super.addCost(cost);
parentPort.postMessage({
type: 'cost',
cost
});
}
}
// Main worker execution
async function runTestGroup() {
const { groupName, testModules, sharedContext, logDir, testRunId } = workerData;
const logger = new WorkerLogger(logDir, testRunId, groupName);
const helpers = new TestHelpers(logger);
logger.info(`Worker started for test group: ${groupName}`);
const results = {
group: groupName,
status: 'passed',
tests: {},
errors: [],
startTime: Date.now()
};
const { groupName, testModules, sharedContext, logDir, testRunId } =
workerData;
try {
// Run each test module in the group
for (const testModule of testModules) {
try {
logger.info(`Running test: ${testModule}`);
// Dynamic import of test module
const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`);
const { default: testFn } = await import(testPath);
// Run the test with shared context
const testResults = await testFn(logger, helpers, sharedContext);
results.tests[testModule] = testResults;
if (testResults.status !== 'passed') {
results.status = 'failed';
if (testResults.errors) {
results.errors.push(...testResults.errors);
}
}
} catch (error) {
logger.error(`Test ${testModule} failed: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) {
logger.error(`Worker error: ${error.message}`);
results.status = 'failed';
results.errors.push({
group: groupName,
error: error.message,
stack: error.stack
});
}
const logger = new WorkerLogger(logDir, testRunId, groupName);
const helpers = new TestHelpers(logger);
results.endTime = Date.now();
results.duration = results.endTime - results.startTime;
// Flush logs and get summary
logger.flush();
const summary = logger.getSummary();
results.summary = summary;
logger.info(`Worker started for test group: ${groupName}`);
// Send results to parent
parentPort.postMessage({
type: 'results',
results
});
const results = {
group: groupName,
status: 'passed',
tests: {},
errors: [],
startTime: Date.now()
};
logger.info(`Worker completed for test group: ${groupName}`);
try {
// Run each test module in the group
for (const testModule of testModules) {
try {
logger.info(`Running test: ${testModule}`);
// Dynamic import of test module
const testPath = join(
dirname(__dirname),
'tests',
`${testModule}.test.js`
);
const { default: testFn } = await import(testPath);
// Run the test with shared context
const testResults = await testFn(logger, helpers, sharedContext);
results.tests[testModule] = testResults;
if (testResults.status !== 'passed') {
results.status = 'failed';
if (testResults.errors) {
results.errors.push(...testResults.errors);
}
}
} catch (error) {
logger.error(`Test ${testModule} failed: ${error.message}`);
results.tests[testModule] = {
status: 'failed',
error: error.message,
stack: error.stack
};
results.status = 'failed';
results.errors.push({
test: testModule,
error: error.message
});
}
}
} catch (error) {
logger.error(`Worker error: ${error.message}`);
results.status = 'failed';
results.errors.push({
group: groupName,
error: error.message,
stack: error.stack
});
}
results.endTime = Date.now();
results.duration = results.endTime - results.startTime;
// Flush logs and get summary
logger.flush();
const summary = logger.getSummary();
results.summary = summary;
// Send results to parent
parentPort.postMessage({
type: 'results',
results
});
logger.info(`Worker completed for test group: ${groupName}`);
}
// Run the test group
runTestGroup().catch(error => {
console.error('Worker fatal error:', error);
process.exit(1);
});
runTestGroup().catch((error) => {
console.error('Worker fatal error:', error);
process.exit(1);
});

675
tests/e2e/tests/advanced.test.js
View File

@@ -1,316 +1,443 @@
export default async function testAdvanced(logger, helpers, context) {
const { testDir } = context;
logger.info('Starting advanced features tests...');
const results = [];
const { testDir } = context;
logger.info('Starting advanced features tests...');
const results = [];
try {
// Test Tag Context Management
logger.info('Testing tag context management...');
// Create new tag contexts
const tag1Result = await helpers.taskMaster('add-tag', ['feature-auth', '--description', 'Authentication feature'], { cwd: testDir });
results.push({
test: 'Create tag context - feature-auth',
passed: tag1Result.exitCode === 0,
output: tag1Result.stdout
});
try {
// Test Tag Context Management
logger.info('Testing tag context management...');
const tag2Result = await helpers.taskMaster('add-tag', ['feature-api', '--description', 'API development'], { cwd: testDir });
results.push({
test: 'Create tag context - feature-api',
passed: tag2Result.exitCode === 0,
output: tag2Result.stdout
});
// Create new tag contexts
const tag1Result = await helpers.taskMaster(
'add-tag',
['feature-auth', '--description', 'Authentication feature'],
{ cwd: testDir }
);
results.push({
test: 'Create tag context - feature-auth',
passed: tag1Result.exitCode === 0,
output: tag1Result.stdout
});
// Add task to feature-auth tag
const task1Result = await helpers.taskMaster('add-task', ['--tag=feature-auth', '--prompt', 'Implement user authentication'], { cwd: testDir });
results.push({
test: 'Add task to feature-auth tag',
passed: task1Result.exitCode === 0,
output: task1Result.stdout
});
const tag2Result = await helpers.taskMaster(
'add-tag',
['feature-api', '--description', 'API development'],
{ cwd: testDir }
);
results.push({
test: 'Create tag context - feature-api',
passed: tag2Result.exitCode === 0,
output: tag2Result.stdout
});
// Add task to feature-api tag
const task2Result = await helpers.taskMaster('add-task', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'], { cwd: testDir });
results.push({
test: 'Add task to feature-api tag',
passed: task2Result.exitCode === 0,
output: task2Result.stdout
});
// Add task to feature-auth tag
const task1Result = await helpers.taskMaster(
'add-task',
['--tag=feature-auth', '--prompt', 'Implement user authentication'],
{ cwd: testDir }
);
results.push({
test: 'Add task to feature-auth tag',
passed: task1Result.exitCode === 0,
output: task1Result.stdout
});
// List all tag contexts
const listTagsResult = await helpers.taskMaster('tags', [], { cwd: testDir });
results.push({
test: 'List all tag contexts',
passed: listTagsResult.exitCode === 0 &&
listTagsResult.stdout.includes('feature-auth') &&
listTagsResult.stdout.includes('feature-api'),
output: listTagsResult.stdout
});
// Add task to feature-api tag
const task2Result = await helpers.taskMaster(
'add-task',
['--tag=feature-api', '--prompt', 'Create REST API endpoints'],
{ cwd: testDir }
);
results.push({
test: 'Add task to feature-api tag',
passed: task2Result.exitCode === 0,
output: task2Result.stdout
});
// List tasks in feature-auth tag
const taggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-auth'], { cwd: testDir });
results.push({
test: 'List tasks in feature-auth tag',
passed: taggedTasksResult.exitCode === 0 &&
taggedTasksResult.stdout.includes('Implement user authentication'),
output: taggedTasksResult.stdout
});
// List all tag contexts
const listTagsResult = await helpers.taskMaster('tags', [], {
cwd: testDir
});
results.push({
test: 'List all tag contexts',
passed:
listTagsResult.exitCode === 0 &&
listTagsResult.stdout.includes('feature-auth') &&
listTagsResult.stdout.includes('feature-api'),
output: listTagsResult.stdout
});
// Test Model Configuration
logger.info('Testing model configuration...');
// List tasks in feature-auth tag
const taggedTasksResult = await helpers.taskMaster(
'list',
['--tag=feature-auth'],
{ cwd: testDir }
);
results.push({
test: 'List tasks in feature-auth tag',
passed:
taggedTasksResult.exitCode === 0 &&
taggedTasksResult.stdout.includes('Implement user authentication'),
output: taggedTasksResult.stdout
});
// Set main model
const setMainModelResult = await helpers.taskMaster('models', ['--set-main', 'gpt-4'], { cwd: testDir });
results.push({
test: 'Set main model',
passed: setMainModelResult.exitCode === 0,
output: setMainModelResult.stdout
});
// Test Model Configuration
logger.info('Testing model configuration...');
// Set research model
const setResearchModelResult = await helpers.taskMaster('models', ['--set-research', 'claude-3-sonnet'], { cwd: testDir });
results.push({
test: 'Set research model',
passed: setResearchModelResult.exitCode === 0,
output: setResearchModelResult.stdout
});
// Set main model
const setMainModelResult = await helpers.taskMaster(
'models',
['--set-main', 'gpt-4'],
{ cwd: testDir }
);
results.push({
test: 'Set main model',
passed: setMainModelResult.exitCode === 0,
output: setMainModelResult.stdout
});
// Set fallback model
const setFallbackModelResult = await helpers.taskMaster('models', ['--set-fallback', 'gpt-3.5-turbo'], { cwd: testDir });
results.push({
test: 'Set fallback model',
passed: setFallbackModelResult.exitCode === 0,
output: setFallbackModelResult.stdout
});
// Set research model
const setResearchModelResult = await helpers.taskMaster(
'models',
['--set-research', 'claude-3-sonnet'],
{ cwd: testDir }
);
results.push({
test: 'Set research model',
passed: setResearchModelResult.exitCode === 0,
output: setResearchModelResult.stdout
});
// Verify model configuration
const showModelsResult = await helpers.taskMaster('models', [], { cwd: testDir });
results.push({
test: 'Show model configuration',
passed: showModelsResult.exitCode === 0 &&
showModelsResult.stdout.includes('gpt-4') &&
showModelsResult.stdout.includes('claude-3-sonnet') &&
showModelsResult.stdout.includes('gpt-3.5-turbo'),
output: showModelsResult.stdout
});
// Set fallback model
const setFallbackModelResult = await helpers.taskMaster(
'models',
['--set-fallback', 'gpt-3.5-turbo'],
{ cwd: testDir }
);
results.push({
test: 'Set fallback model',
passed: setFallbackModelResult.exitCode === 0,
output: setFallbackModelResult.stdout
});
// Test Task Expansion
logger.info('Testing task expansion...');
// Verify model configuration
const showModelsResult = await helpers.taskMaster('models', [], {
cwd: testDir
});
results.push({
test: 'Show model configuration',
passed:
showModelsResult.exitCode === 0 &&
showModelsResult.stdout.includes('gpt-4') &&
showModelsResult.stdout.includes('claude-3-sonnet') &&
showModelsResult.stdout.includes('gpt-3.5-turbo'),
output: showModelsResult.stdout
});
// Add task for expansion
const expandTaskResult = await helpers.taskMaster('add-task', ['--prompt', 'Build REST API'], { cwd: testDir });
const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/);
const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null;
results.push({
test: 'Add task for expansion',
passed: expandTaskResult.exitCode === 0 && expandTaskId !== null,
output: expandTaskResult.stdout
});
// Test Task Expansion
logger.info('Testing task expansion...');
if (expandTaskId) {
// Single task expansion
const expandResult = await helpers.taskMaster('expand', [expandTaskId], { cwd: testDir });
results.push({
test: 'Expand single task',
passed: expandResult.exitCode === 0 && expandResult.stdout.includes('subtasks'),
output: expandResult.stdout
});
// Add task for expansion
const expandTaskResult = await helpers.taskMaster(
'add-task',
['--prompt', 'Build REST API'],
{ cwd: testDir }
);
const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/);
const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null;
// Verify expand worked
const afterExpandResult = await helpers.taskMaster('show', [expandTaskId], { cwd: testDir });
results.push({
test: 'Verify task expansion',
passed: afterExpandResult.exitCode === 0 && afterExpandResult.stdout.includes('subtasks'),
output: afterExpandResult.stdout
});
results.push({
test: 'Add task for expansion',
passed: expandTaskResult.exitCode === 0 && expandTaskId !== null,
output: expandTaskResult.stdout
});
// Force expand (re-expand)
const forceExpandResult = await helpers.taskMaster('expand', [expandTaskId, '--force'], { cwd: testDir });
results.push({
test: 'Force expand task',
passed: forceExpandResult.exitCode === 0,
output: forceExpandResult.stdout
});
}
if (expandTaskId) {
// Single task expansion
const expandResult = await helpers.taskMaster('expand', [expandTaskId], {
cwd: testDir
});
results.push({
test: 'Expand single task',
passed:
expandResult.exitCode === 0 &&
expandResult.stdout.includes('subtasks'),
output: expandResult.stdout
});
// Test Subtask Management
logger.info('Testing subtask management...');
// Verify expand worked
const afterExpandResult = await helpers.taskMaster(
'show',
[expandTaskId],
{ cwd: testDir }
);
results.push({
test: 'Verify task expansion',
passed:
afterExpandResult.exitCode === 0 &&
afterExpandResult.stdout.includes('subtasks'),
output: afterExpandResult.stdout
});
// Add task for subtask testing
const subtaskParentResult = await helpers.taskMaster('add-task', ['--prompt', 'Create user interface'], { cwd: testDir });
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/);
const parentTaskId = parentMatch ? parentMatch[1] : null;
// Force expand (re-expand)
const forceExpandResult = await helpers.taskMaster(
'expand',
[expandTaskId, '--force'],
{ cwd: testDir }
);
results.push({
test: 'Force expand task',
passed: forceExpandResult.exitCode === 0,
output: forceExpandResult.stdout
});
}
if (parentTaskId) {
// Add subtasks manually
const addSubtask1Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Design mockups'], { cwd: testDir });
results.push({
test: 'Add subtask - Design mockups',
passed: addSubtask1Result.exitCode === 0,
output: addSubtask1Result.stdout
});
// Test Subtask Management
logger.info('Testing subtask management...');
const addSubtask2Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Implement components'], { cwd: testDir });
results.push({
test: 'Add subtask - Implement components',
passed: addSubtask2Result.exitCode === 0,
output: addSubtask2Result.stdout
});
// Add task for subtask testing
const subtaskParentResult = await helpers.taskMaster(
'add-task',
['--prompt', 'Create user interface'],
{ cwd: testDir }
);
const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/);
const parentTaskId = parentMatch ? parentMatch[1] : null;
// List subtasks (use show command to see subtasks)
const listSubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir });
results.push({
test: 'List subtasks',
passed: listSubtasksResult.exitCode === 0 &&
listSubtasksResult.stdout.includes('Design mockups') &&
listSubtasksResult.stdout.includes('Implement components'),
output: listSubtasksResult.stdout
});
if (parentTaskId) {
// Add subtasks manually
const addSubtask1Result = await helpers.taskMaster(
'add-subtask',
['--parent', parentTaskId, '--title', 'Design mockups'],
{ cwd: testDir }
);
results.push({
test: 'Add subtask - Design mockups',
passed: addSubtask1Result.exitCode === 0,
output: addSubtask1Result.stdout
});
// Update subtask
const subtaskId = `${parentTaskId}.1`;
const updateSubtaskResult = await helpers.taskMaster('update-subtask', ['--id', subtaskId, '--prompt', 'Create detailed mockups'], { cwd: testDir });
results.push({
test: 'Update subtask',
passed: updateSubtaskResult.exitCode === 0,
output: updateSubtaskResult.stdout
});
const addSubtask2Result = await helpers.taskMaster(
'add-subtask',
['--parent', parentTaskId, '--title', 'Implement components'],
{ cwd: testDir }
);
results.push({
test: 'Add subtask - Implement components',
passed: addSubtask2Result.exitCode === 0,
output: addSubtask2Result.stdout
});
// Remove subtask
const removeSubtaskId = `${parentTaskId}.2`;
const removeSubtaskResult = await helpers.taskMaster('remove-subtask', ['--id', removeSubtaskId], { cwd: testDir });
results.push({
test: 'Remove subtask',
passed: removeSubtaskResult.exitCode === 0,
output: removeSubtaskResult.stdout
});
// List subtasks (use show command to see subtasks)
const listSubtasksResult = await helpers.taskMaster(
'show',
[parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'List subtasks',
passed:
listSubtasksResult.exitCode === 0 &&
listSubtasksResult.stdout.includes('Design mockups') &&
listSubtasksResult.stdout.includes('Implement components'),
output: listSubtasksResult.stdout
});
// Verify subtask changes
const verifySubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir });
results.push({
test: 'Verify subtask changes',
passed: verifySubtasksResult.exitCode === 0 &&
verifySubtasksResult.stdout.includes('Create detailed mockups') &&
!verifySubtasksResult.stdout.includes('Implement components'),
output: verifySubtasksResult.stdout
});
// Update subtask
const subtaskId = `${parentTaskId}.1`;
const updateSubtaskResult = await helpers.taskMaster(
'update-subtask',
['--id', subtaskId, '--prompt', 'Create detailed mockups'],
{ cwd: testDir }
);
results.push({
test: 'Update subtask',
passed: updateSubtaskResult.exitCode === 0,
output: updateSubtaskResult.stdout
});
// Clear all subtasks
const clearSubtasksResult = await helpers.taskMaster('clear-subtasks', ['--id', parentTaskId], { cwd: testDir });
results.push({
test: 'Clear all subtasks',
passed: clearSubtasksResult.exitCode === 0,
output: clearSubtasksResult.stdout
});
// Remove subtask
const removeSubtaskId = `${parentTaskId}.2`;
const removeSubtaskResult = await helpers.taskMaster(
'remove-subtask',
['--id', removeSubtaskId],
{ cwd: testDir }
);
results.push({
test: 'Remove subtask',
passed: removeSubtaskResult.exitCode === 0,
output: removeSubtaskResult.stdout
});
// Verify subtasks cleared
const verifyClearResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir });
results.push({
test: 'Verify subtasks cleared',
passed: verifyClearResult.exitCode === 0 &&
(!verifyClearResult.stdout.includes('Design mockups') &&
!verifyClearResult.stdout.includes('Create detailed mockups')),
output: verifyClearResult.stdout
});
}
// Verify subtask changes
const verifySubtasksResult = await helpers.taskMaster(
'show',
[parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'Verify subtask changes',
passed:
verifySubtasksResult.exitCode === 0 &&
verifySubtasksResult.stdout.includes('Create detailed mockups') &&
!verifySubtasksResult.stdout.includes('Implement components'),
output: verifySubtasksResult.stdout
});
// Test Expand All
logger.info('Testing expand all...');
// Clear all subtasks
const clearSubtasksResult = await helpers.taskMaster(
'clear-subtasks',
['--id', parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'Clear all subtasks',
passed: clearSubtasksResult.exitCode === 0,
output: clearSubtasksResult.stdout
});
// Add multiple tasks
await helpers.taskMaster('add-task', ['--prompt', 'Task A for expand all'], { cwd: testDir });
await helpers.taskMaster('add-task', ['--prompt', 'Task B for expand all'], { cwd: testDir });
// Verify subtasks cleared
const verifyClearResult = await helpers.taskMaster(
'show',
[parentTaskId],
{ cwd: testDir }
);
results.push({
test: 'Verify subtasks cleared',
passed:
verifyClearResult.exitCode === 0 &&
!verifyClearResult.stdout.includes('Design mockups') &&
!verifyClearResult.stdout.includes('Create detailed mockups'),
output: verifyClearResult.stdout
});
}
const expandAllResult = await helpers.taskMaster('expand', ['--all'], { cwd: testDir });
results.push({
test: 'Expand all tasks',
passed: expandAllResult.exitCode === 0,
output: expandAllResult.stdout
});
// Test Expand All
logger.info('Testing expand all...');
// Test Generate Task Files
logger.info('Testing generate task files...');
// Add multiple tasks
await helpers.taskMaster(
'add-task',
['--prompt', 'Task A for expand all'],
{ cwd: testDir }
);
await helpers.taskMaster(
'add-task',
['--prompt', 'Task B for expand all'],
{ cwd: testDir }
);
// Generate files for a specific task
if (expandTaskId) {
const generateResult = await helpers.taskMaster('generate', [expandTaskId], { cwd: testDir });
results.push({
test: 'Generate task files',
passed: generateResult.exitCode === 0,
output: generateResult.stdout
});
const expandAllResult = await helpers.taskMaster('expand', ['--all'], {
cwd: testDir
});
results.push({
test: 'Expand all tasks',
passed: expandAllResult.exitCode === 0,
output: expandAllResult.stdout
});
// Check if files were created
const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`;
const fileExists = helpers.fileExists(taskFilePath);
results.push({
test: 'Verify generated task file exists',
passed: fileExists,
output: fileExists ? `Task file created at ${taskFilePath}` : 'Task file not found'
});
}
// Test Generate Task Files
logger.info('Testing generate task files...');
// Test Tag Context Integrity After Operations
logger.info('Testing tag context integrity after operations...');
// Generate files for a specific task
if (expandTaskId) {
const generateResult = await helpers.taskMaster(
'generate',
[expandTaskId],
{ cwd: testDir }
);
results.push({
test: 'Generate task files',
passed: generateResult.exitCode === 0,
output: generateResult.stdout
});
// Verify tag contexts still exist
const finalTagListResult = await helpers.taskMaster('tags', [], { cwd: testDir });
results.push({
test: 'Final tag context list verification',
passed: finalTagListResult.exitCode === 0 &&
finalTagListResult.stdout.includes('feature-auth') &&
finalTagListResult.stdout.includes('feature-api'),
output: finalTagListResult.stdout
});
// Check if files were created
const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`;
const fileExists = helpers.fileExists(taskFilePath);
// Verify tasks are still in their respective tag contexts
const finalTaggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-api'], { cwd: testDir });
results.push({
test: 'Final tasks in tag context verification',
passed: finalTaggedTasksResult.exitCode === 0 &&
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'),
output: finalTaggedTasksResult.stdout
});
results.push({
test: 'Verify generated task file exists',
passed: fileExists,
output: fileExists
? `Task file created at ${taskFilePath}`
: 'Task file not found'
});
}
// Test Additional Advanced Features
logger.info('Testing additional advanced features...');
// Test Tag Context Integrity After Operations
logger.info('Testing tag context integrity after operations...');
// Test priority task
const priorityTagResult = await helpers.taskMaster('add-task', ['--prompt', 'High priority task', '--priority', 'high'], { cwd: testDir });
results.push({
test: 'Add task with high priority',
passed: priorityTagResult.exitCode === 0,
output: priorityTagResult.stdout
});
// Verify tag contexts still exist
const finalTagListResult = await helpers.taskMaster('tags', [], {
cwd: testDir
});
results.push({
test: 'Final tag context list verification',
passed:
finalTagListResult.exitCode === 0 &&
finalTagListResult.stdout.includes('feature-auth') &&
finalTagListResult.stdout.includes('feature-api'),
output: finalTagListResult.stdout
});
// Test filtering by status
const statusFilterResult = await helpers.taskMaster('list', ['--status', 'pending'], { cwd: testDir });
results.push({
test: 'Filter by status',
passed: statusFilterResult.exitCode === 0,
output: statusFilterResult.stdout
});
// Verify tasks are still in their respective tag contexts
const finalTaggedTasksResult = await helpers.taskMaster(
'list',
['--tag=feature-api'],
{ cwd: testDir }
);
results.push({
test: 'Final tasks in tag context verification',
passed:
finalTaggedTasksResult.exitCode === 0 &&
finalTaggedTasksResult.stdout.includes('Create REST API endpoints'),
output: finalTaggedTasksResult.stdout
});
} catch (error) {
logger.error('Error in advanced features tests:', error);
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
// Test Additional Advanced Features
logger.info('Testing additional advanced features...');
const passed = results.filter(r => r.passed).length;
const total = results.length;
// Test priority task
const priorityTagResult = await helpers.taskMaster(
'add-task',
['--prompt', 'High priority task', '--priority', 'high'],
{ cwd: testDir }
);
results.push({
test: 'Add task with high priority',
passed: priorityTagResult.exitCode === 0,
output: priorityTagResult.stdout
});
return {
name: 'Advanced Features',
passed,
total,
results,
summary: `Advanced features tests: ${passed}/${total} passed`
};
};
// Test filtering by status
const statusFilterResult = await helpers.taskMaster(
'list',
['--status', 'pending'],
{ cwd: testDir }
);
results.push({
test: 'Filter by status',
passed: statusFilterResult.exitCode === 0,
output: statusFilterResult.stdout
});
} catch (error) {
logger.error('Error in advanced features tests:', error);
results.push({
test: 'Advanced features test suite',
passed: false,
error: error.message
});
}
const passed = results.filter((r) => r.passed).length;
const total = results.length;
return {
name: 'Advanced Features',
passed,
total,
results,
summary: `Advanced features tests: ${passed}/${total} passed`
};
}

598
tests/e2e/tests/core.test.js
View File

@@ -4,282 +4,376 @@
*/
export default async function testCoreOperations(logger, helpers, context) {
const { testDir } = context;
const results = {
status: 'passed',
errors: []
};
const { testDir } = context;
const results = {
status: 'passed',
errors: []
};
try {
logger.info('Starting core task operations tests...');
try {
logger.info('Starting core task operations tests...');
// Test 1: List tasks (may have tasks from PRD parsing)
logger.info('\nTest 1: List tasks');
const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult1.exitCode !== 0) {
throw new Error(`List command failed: ${listResult1.stderr}`);
}
// Check for expected output patterns - either empty or with tasks
const hasValidOutput = listResult1.stdout.includes('No tasks found') ||
listResult1.stdout.includes('Task List') ||
listResult1.stdout.includes('Project Dashboard') ||
listResult1.stdout.includes('Listing tasks from');
if (!hasValidOutput) {
throw new Error('Unexpected list output format');
}
logger.success('✓ List tasks successful');
// Test 1: List tasks (may have tasks from PRD parsing)
logger.info('\nTest 1: List tasks');
const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult1.exitCode !== 0) {
throw new Error(`List command failed: ${listResult1.stderr}`);
}
// Check for expected output patterns - either empty or with tasks
const hasValidOutput =
listResult1.stdout.includes('No tasks found') ||
listResult1.stdout.includes('Task List') ||
listResult1.stdout.includes('Project Dashboard') ||
listResult1.stdout.includes('Listing tasks from');
if (!hasValidOutput) {
throw new Error('Unexpected list output format');
}
logger.success('✓ List tasks successful');
// Test 2: Add manual task
logger.info('\nTest 2: Add manual task');
const addResult1 = await helpers.taskMaster('add-task', ['--title', 'Write unit tests', '--description', 'Create comprehensive unit tests for the application'], { cwd: testDir });
if (addResult1.exitCode !== 0) {
throw new Error(`Failed to add manual task: ${addResult1.stderr}`);
}
const manualTaskId = helpers.extractTaskId(addResult1.stdout);
if (!manualTaskId) {
throw new Error('Failed to extract task ID from add output');
}
logger.success(`✓ Added manual task with ID: ${manualTaskId}`);
// Test 2: Add manual task
logger.info('\nTest 2: Add manual task');
const addResult1 = await helpers.taskMaster(
'add-task',
[
'--title',
'Write unit tests',
'--description',
'Create comprehensive unit tests for the application'
],
{ cwd: testDir }
);
if (addResult1.exitCode !== 0) {
throw new Error(`Failed to add manual task: ${addResult1.stderr}`);
}
const manualTaskId = helpers.extractTaskId(addResult1.stdout);
if (!manualTaskId) {
throw new Error('Failed to extract task ID from add output');
}
logger.success(`✓ Added manual task with ID: ${manualTaskId}`);
// Test 3: Add AI task
logger.info('\nTest 3: Add AI task');
const addResult2 = await helpers.taskMaster('add-task', ['--prompt', 'Implement authentication system'], { cwd: testDir });
if (addResult2.exitCode !== 0) {
throw new Error(`Failed to add AI task: ${addResult2.stderr}`);
}
const aiTaskId = helpers.extractTaskId(addResult2.stdout);
if (!aiTaskId) {
throw new Error('Failed to extract AI task ID from add output');
}
logger.success(`✓ Added AI task with ID: ${aiTaskId}`);
// Test 3: Add AI task
logger.info('\nTest 3: Add AI task');
const addResult2 = await helpers.taskMaster(
'add-task',
['--prompt', 'Implement authentication system'],
{ cwd: testDir }
);
if (addResult2.exitCode !== 0) {
throw new Error(`Failed to add AI task: ${addResult2.stderr}`);
}
const aiTaskId = helpers.extractTaskId(addResult2.stdout);
if (!aiTaskId) {
throw new Error('Failed to extract AI task ID from add output');
}
logger.success(`✓ Added AI task with ID: ${aiTaskId}`);
// Test 4: Add another task for dependency testing
logger.info('\nTest 4: Add task for dependency testing');
const addResult3 = await helpers.taskMaster('add-task', ['--title', 'Create database schema', '--description', 'Design and implement the database schema'], { cwd: testDir });
if (addResult3.exitCode !== 0) {
throw new Error(`Failed to add database task: ${addResult3.stderr}`);
}
const dbTaskId = helpers.extractTaskId(addResult3.stdout);
if (!dbTaskId) {
throw new Error('Failed to extract database task ID');
}
logger.success(`✓ Added database task with ID: ${dbTaskId}`);
// Test 4: Add another task for dependency testing
logger.info('\nTest 4: Add task for dependency testing');
const addResult3 = await helpers.taskMaster(
'add-task',
[
'--title',
'Create database schema',
'--description',
'Design and implement the database schema'
],
{ cwd: testDir }
);
if (addResult3.exitCode !== 0) {
throw new Error(`Failed to add database task: ${addResult3.stderr}`);
}
const dbTaskId = helpers.extractTaskId(addResult3.stdout);
if (!dbTaskId) {
throw new Error('Failed to extract database task ID');
}
logger.success(`✓ Added database task with ID: ${dbTaskId}`);
// Test 5: List tasks (should show our newly added tasks)
logger.info('\nTest 5: List all tasks');
const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult2.exitCode !== 0) {
throw new Error(`List command failed: ${listResult2.stderr}`);
}
// Check that we can find our task IDs in the output
const hasTask11 = listResult2.stdout.includes('11');
const hasTask12 = listResult2.stdout.includes('12');
const hasTask13 = listResult2.stdout.includes('13');
if (!hasTask11 || !hasTask12 || !hasTask13) {
throw new Error('Not all task IDs found in list output');
}
// Also check for partial matches (list may truncate titles)
const hasOurTasks = listResult2.stdout.includes('Write') ||
listResult2.stdout.includes('Create');
if (hasOurTasks) {
logger.success('✓ List tasks shows our added tasks');
} else {
logger.warning('Task titles may be truncated in list view');
}
// Test 5: List tasks (should show our newly added tasks)
logger.info('\nTest 5: List all tasks');
const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult2.exitCode !== 0) {
throw new Error(`List command failed: ${listResult2.stderr}`);
}
// Check that we can find our task IDs in the output
const hasTask11 = listResult2.stdout.includes('11');
const hasTask12 = listResult2.stdout.includes('12');
const hasTask13 = listResult2.stdout.includes('13');
// Test 6: Get next task
logger.info('\nTest 6: Get next task');
const nextResult = await helpers.taskMaster('next', [], { cwd: testDir });
if (nextResult.exitCode !== 0) {
throw new Error(`Next task command failed: ${nextResult.stderr}`);
}
logger.success('✓ Get next task successful');
if (!hasTask11 || !hasTask12 || !hasTask13) {
throw new Error('Not all task IDs found in list output');
}
// Test 7: Show task details
logger.info('\nTest 7: Show task details');
const showResult = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir });
if (showResult.exitCode !== 0) {
throw new Error(`Show task details failed: ${showResult.stderr}`);
}
// Check that the task ID is shown and basic structure is present
if (!showResult.stdout.includes(`Task: #${aiTaskId}`) && !showResult.stdout.includes(`ID: │ ${aiTaskId}`)) {
throw new Error('Task ID not found in show output');
}
if (!showResult.stdout.includes('Status:') || !showResult.stdout.includes('Priority:')) {
throw new Error('Task details missing expected fields');
}
logger.success('✓ Show task details successful');
// Also check for partial matches (list may truncate titles)
const hasOurTasks =
listResult2.stdout.includes('Write') ||
listResult2.stdout.includes('Create');
if (hasOurTasks) {
logger.success('✓ List tasks shows our added tasks');
} else {
logger.warning('Task titles may be truncated in list view');
}
// Test 8: Add dependencies
logger.info('\nTest 8: Add dependencies');
const addDepResult = await helpers.taskMaster('add-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir });
if (addDepResult.exitCode !== 0) {
throw new Error(`Failed to add dependency: ${addDepResult.stderr}`);
}
logger.success('✓ Added dependency successfully');
// Test 6: Get next task
logger.info('\nTest 6: Get next task');
const nextResult = await helpers.taskMaster('next', [], { cwd: testDir });
if (nextResult.exitCode !== 0) {
throw new Error(`Next task command failed: ${nextResult.stderr}`);
}
logger.success('✓ Get next task successful');
// Test 9: Verify dependency was added
logger.info('\nTest 9: Verify dependency');
const showResult2 = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir });
if (showResult2.exitCode !== 0) {
throw new Error(`Show task failed: ${showResult2.stderr}`);
}
if (!showResult2.stdout.includes('Dependencies:') || !showResult2.stdout.includes(dbTaskId)) {
throw new Error('Dependency not shown in task details');
}
logger.success('✓ Dependency verified in task details');
// Test 7: Show task details
logger.info('\nTest 7: Show task details');
const showResult = await helpers.taskMaster('show', [aiTaskId], {
cwd: testDir
});
if (showResult.exitCode !== 0) {
throw new Error(`Show task details failed: ${showResult.stderr}`);
}
// Check that the task ID is shown and basic structure is present
if (
!showResult.stdout.includes(`Task: #${aiTaskId}`) &&
!showResult.stdout.includes(`ID: │ ${aiTaskId}`)
) {
throw new Error('Task ID not found in show output');
}
if (
!showResult.stdout.includes('Status:') ||
!showResult.stdout.includes('Priority:')
) {
throw new Error('Task details missing expected fields');
}
logger.success('✓ Show task details successful');
// Test 10: Test circular dependency (should fail)
logger.info('\nTest 10: Test circular dependency prevention');
const circularResult = await helpers.taskMaster('add-dependency', ['--id', dbTaskId, '--depends-on', aiTaskId], {
cwd: testDir,
allowFailure: true
});
if (circularResult.exitCode === 0) {
throw new Error('Circular dependency was not prevented');
}
if (!circularResult.stderr.toLowerCase().includes('circular')) {
throw new Error('Expected circular dependency error message');
}
logger.success('✓ Circular dependency prevented successfully');
// Test 8: Add dependencies
logger.info('\nTest 8: Add dependencies');
const addDepResult = await helpers.taskMaster(
'add-dependency',
['--id', aiTaskId, '--depends-on', dbTaskId],
{ cwd: testDir }
);
if (addDepResult.exitCode !== 0) {
throw new Error(`Failed to add dependency: ${addDepResult.stderr}`);
}
logger.success('✓ Added dependency successfully');
// Test 11: Test non-existent dependency
logger.info('\nTest 11: Test non-existent dependency');
const nonExistResult = await helpers.taskMaster('add-dependency', ['--id', '99999', '--depends-on', '88888'], {
cwd: testDir,
allowFailure: true
});
if (nonExistResult.exitCode === 0) {
throw new Error('Non-existent dependency was incorrectly allowed');
}
logger.success('✓ Non-existent dependency handled correctly');
// Test 9: Verify dependency was added
logger.info('\nTest 9: Verify dependency');
const showResult2 = await helpers.taskMaster('show', [aiTaskId], {
cwd: testDir
});
if (showResult2.exitCode !== 0) {
throw new Error(`Show task failed: ${showResult2.stderr}`);
}
if (
!showResult2.stdout.includes('Dependencies:') ||
!showResult2.stdout.includes(dbTaskId)
) {
throw new Error('Dependency not shown in task details');
}
logger.success('✓ Dependency verified in task details');
// Test 12: Remove dependency
logger.info('\nTest 12: Remove dependency');
const removeDepResult = await helpers.taskMaster('remove-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir });
if (removeDepResult.exitCode !== 0) {
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`);
}
logger.success('✓ Removed dependency successfully');
// Test 10: Test circular dependency (should fail)
logger.info('\nTest 10: Test circular dependency prevention');
const circularResult = await helpers.taskMaster(
'add-dependency',
['--id', dbTaskId, '--depends-on', aiTaskId],
{
cwd: testDir,
allowFailure: true
}
);
if (circularResult.exitCode === 0) {
throw new Error('Circular dependency was not prevented');
}
if (!circularResult.stderr.toLowerCase().includes('circular')) {
throw new Error('Expected circular dependency error message');
}
logger.success('✓ Circular dependency prevented successfully');
// Test 13: Validate dependencies
logger.info('\nTest 13: Validate dependencies');
const validateResult = await helpers.taskMaster('validate-dependencies', [], { cwd: testDir });
if (validateResult.exitCode !== 0) {
throw new Error(`Dependency validation failed: ${validateResult.stderr}`);
}
logger.success('✓ Dependency validation successful');
// Test 11: Test non-existent dependency
logger.info('\nTest 11: Test non-existent dependency');
const nonExistResult = await helpers.taskMaster(
'add-dependency',
['--id', '99999', '--depends-on', '88888'],
{
cwd: testDir,
allowFailure: true
}
);
if (nonExistResult.exitCode === 0) {
throw new Error('Non-existent dependency was incorrectly allowed');
}
logger.success('✓ Non-existent dependency handled correctly');
// Test 14: Update task description
logger.info('\nTest 14: Update task description');
const updateResult = await helpers.taskMaster('update-task', [manualTaskId, '--description', 'Write comprehensive unit tests'], { cwd: testDir });
if (updateResult.exitCode !== 0) {
throw new Error(`Failed to update task: ${updateResult.stderr}`);
}
logger.success('✓ Updated task description successfully');
// Test 12: Remove dependency
logger.info('\nTest 12: Remove dependency');
const removeDepResult = await helpers.taskMaster(
'remove-dependency',
['--id', aiTaskId, '--depends-on', dbTaskId],
{ cwd: testDir }
);
if (removeDepResult.exitCode !== 0) {
throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`);
}
logger.success('✓ Removed dependency successfully');
// Test 15: Add subtask
logger.info('\nTest 15: Add subtask');
const subtaskResult = await helpers.taskMaster('add-subtask', [manualTaskId, 'Write test for login'], { cwd: testDir });
if (subtaskResult.exitCode !== 0) {
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`);
}
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1';
logger.success(`✓ Added subtask with ID: ${subtaskId}`);
// Test 13: Validate dependencies
logger.info('\nTest 13: Validate dependencies');
const validateResult = await helpers.taskMaster(
'validate-dependencies',
[],
{ cwd: testDir }
);
if (validateResult.exitCode !== 0) {
throw new Error(`Dependency validation failed: ${validateResult.stderr}`);
}
logger.success('✓ Dependency validation successful');
// Test 16: Verify subtask relationship
logger.info('\nTest 16: Verify subtask relationship');
const showResult3 = await helpers.taskMaster('show', [manualTaskId], { cwd: testDir });
if (showResult3.exitCode !== 0) {
throw new Error(`Show task failed: ${showResult3.stderr}`);
}
if (!showResult3.stdout.includes('Subtasks:')) {
throw new Error('Subtasks section not shown in parent task');
}
logger.success('✓ Subtask relationship verified');
// Test 14: Update task description
logger.info('\nTest 14: Update task description');
const updateResult = await helpers.taskMaster(
'update-task',
[manualTaskId, '--description', 'Write comprehensive unit tests'],
{ cwd: testDir }
);
if (updateResult.exitCode !== 0) {
throw new Error(`Failed to update task: ${updateResult.stderr}`);
}
logger.success('✓ Updated task description successfully');
// Test 17: Set task status to in_progress
logger.info('\nTest 17: Set task status to in_progress');
const statusResult1 = await helpers.taskMaster('set-status', [manualTaskId, 'in_progress'], { cwd: testDir });
if (statusResult1.exitCode !== 0) {
throw new Error(`Failed to update task status: ${statusResult1.stderr}`);
}
logger.success('✓ Set task status to in_progress');
// Test 15: Add subtask
logger.info('\nTest 15: Add subtask');
const subtaskResult = await helpers.taskMaster(
'add-subtask',
[manualTaskId, 'Write test for login'],
{ cwd: testDir }
);
if (subtaskResult.exitCode !== 0) {
throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`);
}
const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1';
logger.success(`✓ Added subtask with ID: ${subtaskId}`);
// Test 18: Set task status to completed
logger.info('\nTest 18: Set task status to completed');
const statusResult2 = await helpers.taskMaster('set-status', [dbTaskId, 'completed'], { cwd: testDir });
if (statusResult2.exitCode !== 0) {
throw new Error(`Failed to complete task: ${statusResult2.stderr}`);
}
logger.success('✓ Set task status to completed');
// Test 16: Verify subtask relationship
logger.info('\nTest 16: Verify subtask relationship');
const showResult3 = await helpers.taskMaster('show', [manualTaskId], {
cwd: testDir
});
if (showResult3.exitCode !== 0) {
throw new Error(`Show task failed: ${showResult3.stderr}`);
}
if (!showResult3.stdout.includes('Subtasks:')) {
throw new Error('Subtasks section not shown in parent task');
}
logger.success('✓ Subtask relationship verified');
// Test 19: List tasks with status filter
logger.info('\nTest 19: List tasks by status');
const listStatusResult = await helpers.taskMaster('list', ['--status', 'completed'], { cwd: testDir });
if (listStatusResult.exitCode !== 0) {
throw new Error(`List by status failed: ${listStatusResult.stderr}`);
}
if (!listStatusResult.stdout.includes('Create database schema')) {
throw new Error('Completed task not shown in filtered list');
}
logger.success('✓ List tasks by status successful');
// Test 17: Set task status to in_progress
logger.info('\nTest 17: Set task status to in_progress');
const statusResult1 = await helpers.taskMaster(
'set-status',
[manualTaskId, 'in_progress'],
{ cwd: testDir }
);
if (statusResult1.exitCode !== 0) {
throw new Error(`Failed to update task status: ${statusResult1.stderr}`);
}
logger.success('✓ Set task status to in_progress');
// Test 20: Remove single task
logger.info('\nTest 20: Remove single task');
const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], { cwd: testDir });
if (removeResult1.exitCode !== 0) {
throw new Error(`Failed to remove task: ${removeResult1.stderr}`);
}
logger.success('✓ Removed single task successfully');
// Test 18: Set task status to completed
logger.info('\nTest 18: Set task status to completed');
const statusResult2 = await helpers.taskMaster(
'set-status',
[dbTaskId, 'completed'],
{ cwd: testDir }
);
if (statusResult2.exitCode !== 0) {
throw new Error(`Failed to complete task: ${statusResult2.stderr}`);
}
logger.success('✓ Set task status to completed');
// Test 21: Remove multiple tasks
logger.info('\nTest 21: Remove multiple tasks');
const removeResult2 = await helpers.taskMaster('remove-task', [manualTaskId, aiTaskId], { cwd: testDir });
if (removeResult2.exitCode !== 0) {
throw new Error(`Failed to remove multiple tasks: ${removeResult2.stderr}`);
}
logger.success('✓ Removed multiple tasks successfully');
// Test 19: List tasks with status filter
logger.info('\nTest 19: List tasks by status');
const listStatusResult = await helpers.taskMaster(
'list',
['--status', 'completed'],
{ cwd: testDir }
);
if (listStatusResult.exitCode !== 0) {
throw new Error(`List by status failed: ${listStatusResult.stderr}`);
}
if (!listStatusResult.stdout.includes('Create database schema')) {
throw new Error('Completed task not shown in filtered list');
}
logger.success('✓ List tasks by status successful');
// Test 22: Verify tasks were removed
logger.info('\nTest 22: Verify tasks were removed');
const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult3.exitCode !== 0) {
throw new Error(`List command failed: ${listResult3.stderr}`);
}
// Check that our specific task IDs are no longer in the list
const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(listResult3.stdout);
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(listResult3.stdout);
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(listResult3.stdout);
if (stillHasTask11 || stillHasTask12 || stillHasTask13) {
throw new Error('Removed task IDs still appear in list');
}
logger.success('✓ Verified tasks were removed');
// Test 20: Remove single task
logger.info('\nTest 20: Remove single task');
const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], {
cwd: testDir
});
if (removeResult1.exitCode !== 0) {
throw new Error(`Failed to remove task: ${removeResult1.stderr}`);
}
logger.success('✓ Removed single task successfully');
// Test 23: Fix dependencies (cleanup)
logger.info('\nTest 23: Fix dependencies');
const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], { cwd: testDir });
if (fixDepsResult.exitCode !== 0) {
// Non-critical, just log
logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`);
} else {
logger.success('✓ Fix dependencies command executed');
}
// Test 21: Remove multiple tasks
logger.info('\nTest 21: Remove multiple tasks');
const removeResult2 = await helpers.taskMaster(
'remove-task',
[manualTaskId, aiTaskId],
{ cwd: testDir }
);
if (removeResult2.exitCode !== 0) {
throw new Error(
`Failed to remove multiple tasks: ${removeResult2.stderr}`
);
}
logger.success('✓ Removed multiple tasks successfully');
logger.info('\n✅ All core task operations tests passed!');
// Test 22: Verify tasks were removed
logger.info('\nTest 22: Verify tasks were removed');
const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir });
if (listResult3.exitCode !== 0) {
throw new Error(`List command failed: ${listResult3.stderr}`);
}
// Check that our specific task IDs are no longer in the list
const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(
listResult3.stdout
);
const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(
listResult3.stdout
);
const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(
listResult3.stdout
);
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'core operations',
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
if (stillHasTask11 || stillHasTask12 || stillHasTask13) {
throw new Error('Removed task IDs still appear in list');
}
logger.success('✓ Verified tasks were removed');
return results;
}
// Test 23: Fix dependencies (cleanup)
logger.info('\nTest 23: Fix dependencies');
const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], {
cwd: testDir
});
if (fixDepsResult.exitCode !== 0) {
// Non-critical, just log
logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`);
} else {
logger.success('✓ Fix dependencies command executed');
}
logger.info('\n✅ All core task operations tests passed!');
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'core operations',
error: error.message,
stack: error.stack
});
logger.error(`Core operations test failed: ${error.message}`);
}
return results;
}

302
tests/e2e/tests/providers.test.js
View File

@@ -4,161 +4,183 @@
*/
export default async function testProviders(logger, helpers, context) {
const { testDir, config } = context;
const results = {
status: 'passed',
errors: [],
providerComparison: {},
summary: {
totalProviders: 0,
successfulProviders: 0,
failedProviders: 0,
averageExecutionTime: 0,
successRate: '0%'
}
};
const { testDir, config } = context;
const results = {
status: 'passed',
errors: [],
providerComparison: {},
summary: {
totalProviders: 0,
successfulProviders: 0,
failedProviders: 0,
averageExecutionTime: 0,
successRate: '0%'
}
};
try {
logger.info('Starting multi-provider tests...');
try {
logger.info('Starting multi-provider tests...');
const providers = config.providers;
const standardPrompt = config.prompts.addTask;
results.summary.totalProviders = providers.length;
let totalExecutionTime = 0;
const providers = config.providers;
const standardPrompt = config.prompts.addTask;
// Process providers in batches to avoid rate limits
const batchSize = 3;
for (let i = 0; i < providers.length; i += batchSize) {
const batch = providers.slice(i, i + batchSize);
const batchPromises = batch.map(async (provider) => {
const providerResult = {
status: 'failed',
taskId: null,
executionTime: 0,
subtaskCount: 0,
features: {
hasTitle: false,
hasDescription: false,
hasSubtasks: false,
hasDependencies: false
},
error: null,
taskDetails: null
};
results.summary.totalProviders = providers.length;
let totalExecutionTime = 0;
const startTime = Date.now();
// Process providers in batches to avoid rate limits
const batchSize = 3;
for (let i = 0; i < providers.length; i += batchSize) {
const batch = providers.slice(i, i + batchSize);
try {
logger.info(`\nTesting provider: ${provider.name} with model: ${provider.model}`);
const batchPromises = batch.map(async (provider) => {
const providerResult = {
status: 'failed',
taskId: null,
executionTime: 0,
subtaskCount: 0,
features: {
hasTitle: false,
hasDescription: false,
hasSubtasks: false,
hasDependencies: false
},
error: null,
taskDetails: null
};
// Step 1: Set the main model for this provider
logger.info(`Setting model to ${provider.model}...`);
const setModelResult = await helpers.taskMaster('models', ['--set-main', provider.model], { cwd: testDir });
if (setModelResult.exitCode !== 0) {
throw new Error(`Failed to set model for ${provider.name}: ${setModelResult.stderr}`);
}
const startTime = Date.now();
// Step 2: Execute add-task with standard prompt
logger.info(`Adding task with ${provider.name}...`);
const addTaskArgs = ['--prompt', standardPrompt];
if (provider.flags && provider.flags.length > 0) {
addTaskArgs.push(...provider.flags);
}
const addTaskResult = await helpers.taskMaster('add-task', addTaskArgs, {
cwd: testDir,
timeout: 120000 // 2 minutes timeout for AI tasks
});
if (addTaskResult.exitCode !== 0) {
throw new Error(`Add-task failed: ${addTaskResult.stderr}`);
}
try {
logger.info(
`\nTesting provider: ${provider.name} with model: ${provider.model}`
);
// Step 3: Extract task ID from output
const taskId = helpers.extractTaskId(addTaskResult.stdout);
if (!taskId) {
throw new Error(`Failed to extract task ID from output`);
}
providerResult.taskId = taskId;
logger.success(`✓ Created task ${taskId} with ${provider.name}`);
// Step 1: Set the main model for this provider
logger.info(`Setting model to ${provider.model}...`);
const setModelResult = await helpers.taskMaster(
'models',
['--set-main', provider.model],
{ cwd: testDir }
);
if (setModelResult.exitCode !== 0) {
throw new Error(
`Failed to set model for ${provider.name}: ${setModelResult.stderr}`
);
}
// Step 4: Get task details
const showResult = await helpers.taskMaster('show', [taskId], { cwd: testDir });
if (showResult.exitCode === 0) {
providerResult.taskDetails = showResult.stdout;
// Analyze task features
providerResult.features.hasTitle = showResult.stdout.includes('Title:') ||
showResult.stdout.includes('Task:');
providerResult.features.hasDescription = showResult.stdout.includes('Description:');
providerResult.features.hasSubtasks = showResult.stdout.includes('Subtasks:');
providerResult.features.hasDependencies = showResult.stdout.includes('Dependencies:');
// Count subtasks
const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g);
providerResult.subtaskCount = subtaskMatches ? subtaskMatches.length : 0;
}
// Step 2: Execute add-task with standard prompt
logger.info(`Adding task with ${provider.name}...`);
const addTaskArgs = ['--prompt', standardPrompt];
if (provider.flags && provider.flags.length > 0) {
addTaskArgs.push(...provider.flags);
}
providerResult.status = 'success';
results.summary.successfulProviders++;
const addTaskResult = await helpers.taskMaster(
'add-task',
addTaskArgs,
{
cwd: testDir,
timeout: 120000 // 2 minutes timeout for AI tasks
}
);
} catch (error) {
providerResult.status = 'failed';
providerResult.error = error.message;
results.summary.failedProviders++;
logger.error(`${provider.name} test failed: ${error.message}`);
}
if (addTaskResult.exitCode !== 0) {
throw new Error(`Add-task failed: ${addTaskResult.stderr}`);
}
providerResult.executionTime = Date.now() - startTime;
totalExecutionTime += providerResult.executionTime;
results.providerComparison[provider.name] = providerResult;
});
// Step 3: Extract task ID from output
const taskId = helpers.extractTaskId(addTaskResult.stdout);
if (!taskId) {
throw new Error(`Failed to extract task ID from output`);
}
providerResult.taskId = taskId;
logger.success(`✓ Created task ${taskId} with ${provider.name}`);
// Wait for batch to complete
await Promise.all(batchPromises);
// Small delay between batches to avoid rate limits
if (i + batchSize < providers.length) {
logger.info('Waiting 2 seconds before next batch...');
await helpers.wait(2000);
}
}
// Step 4: Get task details
const showResult = await helpers.taskMaster('show', [taskId], {
cwd: testDir
});
if (showResult.exitCode === 0) {
providerResult.taskDetails = showResult.stdout;
// Calculate summary statistics
results.summary.averageExecutionTime = Math.round(totalExecutionTime / providers.length);
results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`;
// Analyze task features
providerResult.features.hasTitle =
showResult.stdout.includes('Title:') ||
showResult.stdout.includes('Task:');
providerResult.features.hasDescription =
showResult.stdout.includes('Description:');
providerResult.features.hasSubtasks =
showResult.stdout.includes('Subtasks:');
providerResult.features.hasDependencies =
showResult.stdout.includes('Dependencies:');
// Log summary
logger.info('\n=== Provider Test Summary ===');
logger.info(`Total providers tested: ${results.summary.totalProviders}`);
logger.info(`Successful: ${results.summary.successfulProviders}`);
logger.info(`Failed: ${results.summary.failedProviders}`);
logger.info(`Success rate: ${results.summary.successRate}`);
logger.info(`Average execution time: ${results.summary.averageExecutionTime}ms`);
// Count subtasks
const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g);
providerResult.subtaskCount = subtaskMatches
? subtaskMatches.length
: 0;
}
// Determine overall status
if (results.summary.failedProviders === 0) {
logger.success('✅ All provider tests passed!');
} else if (results.summary.successfulProviders > 0) {
results.status = 'partial';
logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`);
} else {
results.status = 'failed';
logger.error('❌ All provider tests failed');
}
providerResult.status = 'success';
results.summary.successfulProviders++;
} catch (error) {
providerResult.status = 'failed';
providerResult.error = error.message;
results.summary.failedProviders++;
logger.error(`${provider.name} test failed: ${error.message}`);
}
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
providerResult.executionTime = Date.now() - startTime;
totalExecutionTime += providerResult.executionTime;
return results;
}
results.providerComparison[provider.name] = providerResult;
});
// Wait for batch to complete
await Promise.all(batchPromises);
// Small delay between batches to avoid rate limits
if (i + batchSize < providers.length) {
logger.info('Waiting 2 seconds before next batch...');
await helpers.wait(2000);
}
}
// Calculate summary statistics
results.summary.averageExecutionTime = Math.round(
totalExecutionTime / providers.length
);
results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`;
// Log summary
logger.info('\n=== Provider Test Summary ===');
logger.info(`Total providers tested: ${results.summary.totalProviders}`);
logger.info(`Successful: ${results.summary.successfulProviders}`);
logger.info(`Failed: ${results.summary.failedProviders}`);
logger.info(`Success rate: ${results.summary.successRate}`);
logger.info(
`Average execution time: ${results.summary.averageExecutionTime}ms`
);
// Determine overall status
if (results.summary.failedProviders === 0) {
logger.success('✅ All provider tests passed!');
} else if (results.summary.successfulProviders > 0) {
results.status = 'partial';
logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`);
} else {
results.status = 'failed';
logger.error('❌ All provider tests failed');
}
} catch (error) {
results.status = 'failed';
results.errors.push({
test: 'provider tests',
error: error.message,
stack: error.stack
});
logger.error(`Provider tests failed: ${error.message}`);
}
return results;
}

429
tests/e2e/tests/setup.test.js
View File

@@ -9,108 +9,119 @@ import { testConfig } from '../config/test-config.js';
* @returns {Promise<Object>} Test results object with status and directory path
*/
export async function runSetupTest(logger, helpers) {
const testResults = {
status: 'pending',
testDir: null,
steps: {
createDirectory: false,
linkGlobally: false,
copyEnv: false,
initialize: false,
parsePrd: false,
analyzeComplexity: false,
generateReport: false
},
errors: [],
prdPath: null,
complexityReport: null
};
const testResults = {
status: 'pending',
testDir: null,
steps: {
createDirectory: false,
linkGlobally: false,
copyEnv: false,
initialize: false,
parsePrd: false,
analyzeComplexity: false,
generateReport: false
},
errors: [],
prdPath: null,
complexityReport: null
};
try {
// Step 1: Create test directory with timestamp
logger.step('Creating test directory');
const timestamp = new Date().toISOString().replace(/[:.]/g, '').replace('T', '_').slice(0, -1);
const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`);
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
testResults.testDir = testDir;
testResults.steps.createDirectory = true;
logger.success(`Test directory created: ${testDir}`);
try {
// Step 1: Create test directory with timestamp
logger.step('Creating test directory');
const timestamp = new Date()
.toISOString()
.replace(/[:.]/g, '')
.replace('T', '_')
.slice(0, -1);
const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`);
// Step 2: Link task-master globally
logger.step('Linking task-master globally');
const linkResult = await helpers.executeCommand('npm', ['link'], {
cwd: testConfig.paths.projectRoot,
timeout: 60000
});
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
if (linkResult.exitCode === 0) {
testResults.steps.linkGlobally = true;
logger.success('Task-master linked globally');
} else {
throw new Error(`Failed to link task-master: ${linkResult.stderr}`);
}
testResults.testDir = testDir;
testResults.steps.createDirectory = true;
logger.success(`Test directory created: ${testDir}`);
// Step 3: Copy .env file
logger.step('Copying .env file to test directory');
const envSourcePath = testConfig.paths.mainEnvFile;
const envDestPath = join(testDir, '.env');
if (helpers.fileExists(envSourcePath)) {
if (helpers.copyFile(envSourcePath, envDestPath)) {
testResults.steps.copyEnv = true;
logger.success('.env file copied successfully');
} else {
throw new Error('Failed to copy .env file');
}
} else {
logger.warning('.env file not found at source, proceeding without it');
}
// Step 2: Link task-master globally
logger.step('Linking task-master globally');
const linkResult = await helpers.executeCommand('npm', ['link'], {
cwd: testConfig.paths.projectRoot,
timeout: 60000
});
// Step 4: Initialize project with task-master init
logger.step('Initializing project with task-master');
const initResult = await helpers.taskMaster('init', [
'-y',
'--name="E2E Test ' + testDir.split('/').pop() + '"',
'--description="Automated E2E test run"'
], {
cwd: testDir,
timeout: 120000
});
if (linkResult.exitCode === 0) {
testResults.steps.linkGlobally = true;
logger.success('Task-master linked globally');
} else {
throw new Error(`Failed to link task-master: ${linkResult.stderr}`);
}
if (initResult.exitCode === 0) {
testResults.steps.initialize = true;
logger.success('Project initialized successfully');
// Save init debug log if available
const initDebugPath = join(testDir, 'init-debug.log');
if (existsSync(initDebugPath)) {
logger.info('Init debug log saved');
}
} else {
throw new Error(`Initialization failed: ${initResult.stderr}`);
}
// Step 3: Copy .env file
logger.step('Copying .env file to test directory');
const envSourcePath = testConfig.paths.mainEnvFile;
const envDestPath = join(testDir, '.env');
// Step 5: Parse PRD from sample file
logger.step('Parsing PRD from sample file');
// First, copy the sample PRD to the test directory
const prdSourcePath = testConfig.paths.samplePrdSource;
const prdDestPath = join(testDir, 'prd.txt');
testResults.prdPath = prdDestPath;
if (!helpers.fileExists(prdSourcePath)) {
// If sample PRD doesn't exist in fixtures, use the example PRD
const examplePrdPath = join(testConfig.paths.projectRoot, 'assets/example_prd.txt');
if (helpers.fileExists(examplePrdPath)) {
helpers.copyFile(examplePrdPath, prdDestPath);
logger.info('Using example PRD file');
} else {
// Create a minimal PRD for testing
const minimalPrd = `<PRD>
if (helpers.fileExists(envSourcePath)) {
if (helpers.copyFile(envSourcePath, envDestPath)) {
testResults.steps.copyEnv = true;
logger.success('.env file copied successfully');
} else {
throw new Error('Failed to copy .env file');
}
} else {
logger.warning('.env file not found at source, proceeding without it');
}
// Step 4: Initialize project with task-master init
logger.step('Initializing project with task-master');
const initResult = await helpers.taskMaster(
'init',
[
'-y',
'--name="E2E Test ' + testDir.split('/').pop() + '"',
'--description="Automated E2E test run"'
],
{
cwd: testDir,
timeout: 120000
}
);
if (initResult.exitCode === 0) {
testResults.steps.initialize = true;
logger.success('Project initialized successfully');
// Save init debug log if available
const initDebugPath = join(testDir, 'init-debug.log');
if (existsSync(initDebugPath)) {
logger.info('Init debug log saved');
}
} else {
throw new Error(`Initialization failed: ${initResult.stderr}`);
}
// Step 5: Parse PRD from sample file
logger.step('Parsing PRD from sample file');
// First, copy the sample PRD to the test directory
const prdSourcePath = testConfig.paths.samplePrdSource;
const prdDestPath = join(testDir, 'prd.txt');
testResults.prdPath = prdDestPath;
if (!helpers.fileExists(prdSourcePath)) {
// If sample PRD doesn't exist in fixtures, use the example PRD
const examplePrdPath = join(
testConfig.paths.projectRoot,
'assets/example_prd.txt'
);
if (helpers.fileExists(examplePrdPath)) {
helpers.copyFile(examplePrdPath, prdDestPath);
logger.info('Using example PRD file');
} else {
// Create a minimal PRD for testing
const minimalPrd = `<PRD>
# Overview
A simple task management system for developers.
@@ -145,122 +156,140 @@ Phase 2: Enhanced features
5. CLI interface
6. Advanced features
</PRD>`;
writeFileSync(prdDestPath, minimalPrd);
logger.info('Created minimal PRD for testing');
}
} else {
helpers.copyFile(prdSourcePath, prdDestPath);
}
// Parse the PRD
const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], {
cwd: testDir,
timeout: 180000
});
writeFileSync(prdDestPath, minimalPrd);
logger.info('Created minimal PRD for testing');
}
} else {
helpers.copyFile(prdSourcePath, prdDestPath);
}
if (parsePrdResult.exitCode === 0) {
testResults.steps.parsePrd = true;
logger.success('PRD parsed successfully');
// Extract task count from output
const taskCountMatch = parsePrdResult.stdout.match(/(\d+) tasks? created/i);
if (taskCountMatch) {
logger.info(`Created ${taskCountMatch[1]} tasks from PRD`);
}
} else {
throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`);
}
// Parse the PRD
const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], {
cwd: testDir,
timeout: 180000
});
// Step 6: Run complexity analysis
logger.step('Running complexity analysis on parsed tasks');
// Ensure reports directory exists
const reportsDir = join(testDir, '.taskmaster/reports');
if (!existsSync(reportsDir)) {
mkdirSync(reportsDir, { recursive: true });
}
const analyzeResult = await helpers.taskMaster('analyze-complexity', ['--research', '--output', '.taskmaster/reports/task-complexity-report.json'], {
cwd: testDir,
timeout: 240000
});
if (parsePrdResult.exitCode === 0) {
testResults.steps.parsePrd = true;
logger.success('PRD parsed successfully');
if (analyzeResult.exitCode === 0) {
testResults.steps.analyzeComplexity = true;
logger.success('Complexity analysis completed');
// Extract complexity information from output
const complexityMatch = analyzeResult.stdout.match(/Total Complexity Score: ([\d.]+)/);
if (complexityMatch) {
logger.info(`Total complexity score: ${complexityMatch[1]}`);
}
} else {
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`);
}
// Extract task count from output
const taskCountMatch =
parsePrdResult.stdout.match(/(\d+) tasks? created/i);
if (taskCountMatch) {
logger.info(`Created ${taskCountMatch[1]} tasks from PRD`);
}
} else {
throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`);
}
// Step 7: Generate complexity report
logger.step('Generating complexity report');
const reportResult = await helpers.taskMaster('complexity-report', [], {
cwd: testDir,
timeout: 60000
});
// Step 6: Run complexity analysis
logger.step('Running complexity analysis on parsed tasks');
// Ensure reports directory exists
const reportsDir = join(testDir, '.taskmaster/reports');
if (!existsSync(reportsDir)) {
mkdirSync(reportsDir, { recursive: true });
}
const analyzeResult = await helpers.taskMaster(
'analyze-complexity',
[
'--research',
'--output',
'.taskmaster/reports/task-complexity-report.json'
],
{
cwd: testDir,
timeout: 240000
}
);
if (reportResult.exitCode === 0) {
testResults.steps.generateReport = true;
logger.success('Complexity report generated');
// Check if complexity report file was created (not needed since complexity-report reads from the standard location)
const reportPath = join(testDir, '.taskmaster/reports/task-complexity-report.json');
if (helpers.fileExists(reportPath)) {
testResults.complexityReport = helpers.readJson(reportPath);
logger.info('Complexity report saved to task-complexity-report.json');
// Log summary if available
if (testResults.complexityReport && testResults.complexityReport.summary) {
const summary = testResults.complexityReport.summary;
logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`);
logger.info(`Average complexity: ${summary.averageComplexity || 0}`);
}
}
} else {
logger.warning(`Complexity report generation had issues: ${reportResult.stderr}`);
// Don't fail the test for report generation issues
testResults.steps.generateReport = true;
}
if (analyzeResult.exitCode === 0) {
testResults.steps.analyzeComplexity = true;
logger.success('Complexity analysis completed');
// Verify tasks.json was created
const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json');
if (helpers.fileExists(tasksJsonPath)) {
const taskCount = helpers.getTaskCount(tasksJsonPath);
logger.info(`Verified tasks.json exists with ${taskCount} tasks`);
} else {
throw new Error('tasks.json was not created');
}
// Extract complexity information from output
const complexityMatch = analyzeResult.stdout.match(
/Total Complexity Score: ([\d.]+)/
);
if (complexityMatch) {
logger.info(`Total complexity score: ${complexityMatch[1]}`);
}
} else {
throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`);
}
// All steps completed successfully
testResults.status = 'success';
logger.success('Setup test completed successfully');
// Step 7: Generate complexity report
logger.step('Generating complexity report');
const reportResult = await helpers.taskMaster('complexity-report', [], {
cwd: testDir,
timeout: 60000
});
} catch (error) {
testResults.status = 'failed';
testResults.errors.push(error.message);
logger.error(`Setup test failed: ${error.message}`);
// Log which steps completed
logger.info('Completed steps:');
Object.entries(testResults.steps).forEach(([step, completed]) => {
if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
if (reportResult.exitCode === 0) {
testResults.steps.generateReport = true;
logger.success('Complexity report generated');
// Flush logs before returning
logger.flush();
// Check if complexity report file was created (not needed since complexity-report reads from the standard location)
const reportPath = join(
testDir,
'.taskmaster/reports/task-complexity-report.json'
);
if (helpers.fileExists(reportPath)) {
testResults.complexityReport = helpers.readJson(reportPath);
logger.info('Complexity report saved to task-complexity-report.json');
return testResults;
// Log summary if available
if (
testResults.complexityReport &&
testResults.complexityReport.summary
) {
const summary = testResults.complexityReport.summary;
logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`);
logger.info(`Average complexity: ${summary.averageComplexity || 0}`);
}
}
} else {
logger.warning(
`Complexity report generation had issues: ${reportResult.stderr}`
);
// Don't fail the test for report generation issues
testResults.steps.generateReport = true;
}
// Verify tasks.json was created
const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json');
if (helpers.fileExists(tasksJsonPath)) {
const taskCount = helpers.getTaskCount(tasksJsonPath);
logger.info(`Verified tasks.json exists with ${taskCount} tasks`);
} else {
throw new Error('tasks.json was not created');
}
// All steps completed successfully
testResults.status = 'success';
logger.success('Setup test completed successfully');
} catch (error) {
testResults.status = 'failed';
testResults.errors.push(error.message);
logger.error(`Setup test failed: ${error.message}`);
// Log which steps completed
logger.info('Completed steps:');
Object.entries(testResults.steps).forEach(([step, completed]) => {
if (completed) {
logger.info(` ✓ ${step}`);
} else {
logger.info(` ✗ ${step}`);
}
});
}
// Flush logs before returning
logger.flush();
return testResults;
}
// Export default for direct execution
export default runSetupTest;
export default runSetupTest;

403
tests/e2e/utils/error-handler.js
View File

@@ -3,234 +3,245 @@ import { join } from 'path';
import chalk from 'chalk';
export class ErrorHandler {
constructor(logger) {
this.logger = logger;
this.errors = [];
this.warnings = [];
}
constructor(logger) {
this.logger = logger;
this.errors = [];
this.warnings = [];
}
/**
* Handle and categorize errors
*/
handleError(error, context = {}) {
const errorInfo = {
timestamp: new Date().toISOString(),
message: error.message || 'Unknown error',
stack: error.stack,
context,
type: this.categorizeError(error)
};
/**
* Handle and categorize errors
*/
handleError(error, context = {}) {
const errorInfo = {
timestamp: new Date().toISOString(),
message: error.message || 'Unknown error',
stack: error.stack,
context,
type: this.categorizeError(error)
};
this.errors.push(errorInfo);
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
this.errors.push(errorInfo);
this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`);
if (context.critical) {
throw error;
}
if (context.critical) {
throw error;
}
return errorInfo;
}
return errorInfo;
}
/**
* Add a warning
*/
addWarning(message, context = {}) {
const warning = {
timestamp: new Date().toISOString(),
message,
context
};
/**
* Add a warning
*/
addWarning(message, context = {}) {
const warning = {
timestamp: new Date().toISOString(),
message,
context
};
this.warnings.push(warning);
this.logger.warning(message);
}
this.warnings.push(warning);
this.logger.warning(message);
}
/**
* Categorize error types
*/
categorizeError(error) {
const message = error.message.toLowerCase();
/**
* Categorize error types
*/
categorizeError(error) {
const message = error.message.toLowerCase();
if (message.includes('command not found') || message.includes('not found')) {
return 'DEPENDENCY_ERROR';
}
if (message.includes('permission') || message.includes('access denied')) {
return 'PERMISSION_ERROR';
}
if (message.includes('timeout')) {
return 'TIMEOUT_ERROR';
}
if (message.includes('api') || message.includes('rate limit')) {
return 'API_ERROR';
}
if (message.includes('json') || message.includes('parse')) {
return 'PARSE_ERROR';
}
if (message.includes('file') || message.includes('directory')) {
return 'FILE_ERROR';
}
if (
message.includes('command not found') ||
message.includes('not found')
) {
return 'DEPENDENCY_ERROR';
}
if (message.includes('permission') || message.includes('access denied')) {
return 'PERMISSION_ERROR';
}
if (message.includes('timeout')) {
return 'TIMEOUT_ERROR';
}
if (message.includes('api') || message.includes('rate limit')) {
return 'API_ERROR';
}
if (message.includes('json') || message.includes('parse')) {
return 'PARSE_ERROR';
}
if (message.includes('file') || message.includes('directory')) {
return 'FILE_ERROR';
}
return 'GENERAL_ERROR';
}
return 'GENERAL_ERROR';
}
/**
* Get error summary
*/
getSummary() {
const errorsByType = {};
this.errors.forEach(error => {
if (!errorsByType[error.type]) {
errorsByType[error.type] = [];
}
errorsByType[error.type].push(error);
});
/**
* Get error summary
*/
getSummary() {
const errorsByType = {};
return {
totalErrors: this.errors.length,
totalWarnings: this.warnings.length,
errorsByType,
criticalErrors: this.errors.filter(e => e.context.critical),
recentErrors: this.errors.slice(-5)
};
}
this.errors.forEach((error) => {
if (!errorsByType[error.type]) {
errorsByType[error.type] = [];
}
errorsByType[error.type].push(error);
});
/**
* Generate error report
*/
generateReport(outputPath) {
const summary = this.getSummary();
const report = {
generatedAt: new Date().toISOString(),
summary: {
totalErrors: summary.totalErrors,
totalWarnings: summary.totalWarnings,
errorTypes: Object.keys(summary.errorsByType)
},
errors: this.errors,
warnings: this.warnings,
recommendations: this.generateRecommendations(summary)
};
return {
totalErrors: this.errors.length,
totalWarnings: this.warnings.length,
errorsByType,
criticalErrors: this.errors.filter((e) => e.context.critical),
recentErrors: this.errors.slice(-5)
};
}
writeFileSync(outputPath, JSON.stringify(report, null, 2));
return report;
}
/**
* Generate error report
*/
generateReport(outputPath) {
const summary = this.getSummary();
const report = {
generatedAt: new Date().toISOString(),
summary: {
totalErrors: summary.totalErrors,
totalWarnings: summary.totalWarnings,
errorTypes: Object.keys(summary.errorsByType)
},
errors: this.errors,
warnings: this.warnings,
recommendations: this.generateRecommendations(summary)
};
/**
* Generate recommendations based on errors
*/
generateRecommendations(summary) {
const recommendations = [];
writeFileSync(outputPath, JSON.stringify(report, null, 2));
return report;
}
if (summary.errorsByType.DEPENDENCY_ERROR) {
recommendations.push({
type: 'DEPENDENCY',
message: 'Install missing dependencies using npm install or check PATH',
errors: summary.errorsByType.DEPENDENCY_ERROR.length
});
}
/**
* Generate recommendations based on errors
*/
generateRecommendations(summary) {
const recommendations = [];
if (summary.errorsByType.PERMISSION_ERROR) {
recommendations.push({
type: 'PERMISSION',
message: 'Check file permissions or run with appropriate privileges',
errors: summary.errorsByType.PERMISSION_ERROR.length
});
}
if (summary.errorsByType.DEPENDENCY_ERROR) {
recommendations.push({
type: 'DEPENDENCY',
message: 'Install missing dependencies using npm install or check PATH',
errors: summary.errorsByType.DEPENDENCY_ERROR.length
});
}
if (summary.errorsByType.API_ERROR) {
recommendations.push({
type: 'API',
message: 'Check API keys, rate limits, or network connectivity',
errors: summary.errorsByType.API_ERROR.length
});
}
if (summary.errorsByType.PERMISSION_ERROR) {
recommendations.push({
type: 'PERMISSION',
message: 'Check file permissions or run with appropriate privileges',
errors: summary.errorsByType.PERMISSION_ERROR.length
});
}
if (summary.errorsByType.TIMEOUT_ERROR) {
recommendations.push({
type: 'TIMEOUT',
message: 'Consider increasing timeout values or optimizing slow operations',
errors: summary.errorsByType.TIMEOUT_ERROR.length
});
}
if (summary.errorsByType.API_ERROR) {
recommendations.push({
type: 'API',
message: 'Check API keys, rate limits, or network connectivity',
errors: summary.errorsByType.API_ERROR.length
});
}
return recommendations;
}
if (summary.errorsByType.TIMEOUT_ERROR) {
recommendations.push({
type: 'TIMEOUT',
message:
'Consider increasing timeout values or optimizing slow operations',
errors: summary.errorsByType.TIMEOUT_ERROR.length
});
}
/**
* Display error summary in console
*/
displaySummary() {
const summary = this.getSummary();
return recommendations;
}
if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
console.log(chalk.green('✅ No errors or warnings detected'));
return;
}
/**
* Display error summary in console
*/
displaySummary() {
const summary = this.getSummary();
console.log(chalk.red.bold(`\n🚨 Error Summary:`));
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
if (summary.totalErrors === 0 && summary.totalWarnings === 0) {
console.log(chalk.green('✅ No errors or warnings detected'));
return;
}
if (summary.totalErrors > 0) {
console.log(chalk.red.bold('\n Error Types:'));
Object.entries(summary.errorsByType).forEach(([type, errors]) => {
console.log(chalk.red(` - ${type}: ${errors.length}`));
});
console.log(chalk.red.bold(`\n🚨 Error Summary:`));
console.log(chalk.red(` Total Errors: ${summary.totalErrors}`));
console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`));
if (summary.criticalErrors.length > 0) {
console.log(chalk.red.bold(`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`));
summary.criticalErrors.forEach(error => {
console.log(chalk.red(` - ${error.message}`));
});
}
}
if (summary.totalErrors > 0) {
console.log(chalk.red.bold('\n Error Types:'));
Object.entries(summary.errorsByType).forEach(([type, errors]) => {
console.log(chalk.red(` - ${type}: ${errors.length}`));
});
const recommendations = this.generateRecommendations(summary);
if (recommendations.length > 0) {
console.log(chalk.yellow.bold('\n💡 Recommendations:'));
recommendations.forEach(rec => {
console.log(chalk.yellow(` - ${rec.message}`));
});
}
}
if (summary.criticalErrors.length > 0) {
console.log(
chalk.red.bold(
`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`
)
);
summary.criticalErrors.forEach((error) => {
console.log(chalk.red(` - ${error.message}`));
});
}
}
/**
* Clear all errors and warnings
*/
clear() {
this.errors = [];
this.warnings = [];
}
const recommendations = this.generateRecommendations(summary);
if (recommendations.length > 0) {
console.log(chalk.yellow.bold('\n💡 Recommendations:'));
recommendations.forEach((rec) => {
console.log(chalk.yellow(` - ${rec.message}`));
});
}
}
/**
* Clear all errors and warnings
*/
clear() {
this.errors = [];
this.warnings = [];
}
}
/**
* Global error handler for uncaught exceptions
*/
export function setupGlobalErrorHandlers(errorHandler, logger) {
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`);
errorHandler.handleError(error, { critical: true, source: 'uncaughtException' });
process.exit(1);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`);
errorHandler.handleError(error, {
critical: true,
source: 'uncaughtException'
});
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
errorHandler.handleError(new Error(String(reason)), {
critical: false,
source: 'unhandledRejection'
});
});
process.on('unhandledRejection', (reason, promise) => {
logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
errorHandler.handleError(new Error(String(reason)), {
critical: false,
source: 'unhandledRejection'
});
});
process.on('SIGINT', () => {
logger.info('\nReceived SIGINT, shutting down gracefully...');
errorHandler.displaySummary();
process.exit(130);
});
process.on('SIGINT', () => {
logger.info('\nReceived SIGINT, shutting down gracefully...');
errorHandler.displaySummary();
process.exit(130);
});
process.on('SIGTERM', () => {
logger.info('\nReceived SIGTERM, shutting down...');
errorHandler.displaySummary();
process.exit(143);
});
}
process.on('SIGTERM', () => {
logger.info('\nReceived SIGTERM, shutting down...');
errorHandler.displaySummary();
process.exit(143);
});
}

223
tests/e2e/utils/llm-analyzer.js
View File

@@ -2,56 +2,58 @@ import { readFileSync } from 'fs';
import fetch from 'node-fetch';
export class LLMAnalyzer {
constructor(config, logger) {
this.config = config;
this.logger = logger;
this.apiKey = process.env.ANTHROPIC_API_KEY;
this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
}
constructor(config, logger) {
this.config = config;
this.logger = logger;
this.apiKey = process.env.ANTHROPIC_API_KEY;
this.apiEndpoint = 'https://api.anthropic.com/v1/messages';
}
async analyzeLog(logFile, providerSummaryFile = null) {
if (!this.config.llmAnalysis.enabled) {
this.logger.info('LLM analysis is disabled in configuration');
return null;
}
async analyzeLog(logFile, providerSummaryFile = null) {
if (!this.config.llmAnalysis.enabled) {
this.logger.info('LLM analysis is disabled in configuration');
return null;
}
if (!this.apiKey) {
this.logger.error('ANTHROPIC_API_KEY not found in environment');
return null;
}
if (!this.apiKey) {
this.logger.error('ANTHROPIC_API_KEY not found in environment');
return null;
}
try {
const logContent = readFileSync(logFile, 'utf8');
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
try {
const logContent = readFileSync(logFile, 'utf8');
const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile);
const response = await this.callLLM(prompt);
const analysis = this.parseResponse(response);
// Calculate and log cost
if (response.usage) {
const cost = this.calculateCost(response.usage);
this.logger.addCost(cost);
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
}
const response = await this.callLLM(prompt);
const analysis = this.parseResponse(response);
return analysis;
} catch (error) {
this.logger.error(`LLM analysis failed: ${error.message}`);
return null;
}
}
// Calculate and log cost
if (response.usage) {
const cost = this.calculateCost(response.usage);
this.logger.addCost(cost);
this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`);
}
buildAnalysisPrompt(logContent, providerSummaryFile) {
let providerSummary = '';
if (providerSummaryFile) {
try {
providerSummary = readFileSync(providerSummaryFile, 'utf8');
} catch (error) {
this.logger.warning(`Could not read provider summary file: ${error.message}`);
}
}
return analysis;
} catch (error) {
this.logger.error(`LLM analysis failed: ${error.message}`);
return null;
}
}
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
buildAnalysisPrompt(logContent, providerSummaryFile) {
let providerSummary = '';
if (providerSummaryFile) {
try {
providerSummary = readFileSync(providerSummaryFile, 'utf8');
} catch (error) {
this.logger.warning(
`Could not read provider summary file: ${error.message}`
);
}
}
return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially.
Your goal is to:
1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files).
@@ -88,81 +90,82 @@ Return your analysis **strictly** in the following JSON format. Do not include a
Here is the main log content:
${logContent}`;
}
}
async callLLM(prompt) {
const payload = {
model: this.config.llmAnalysis.model,
max_tokens: this.config.llmAnalysis.maxTokens,
messages: [
{ role: 'user', content: prompt }
]
};
async callLLM(prompt) {
const payload = {
model: this.config.llmAnalysis.model,
max_tokens: this.config.llmAnalysis.maxTokens,
messages: [{ role: 'user', content: prompt }]
};
const response = await fetch(this.apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01'
},
body: JSON.stringify(payload)
});
const response = await fetch(this.apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01'
},
body: JSON.stringify(payload)
});
if (!response.ok) {
const error = await response.text();
throw new Error(`LLM API call failed: ${response.status} - ${error}`);
}
if (!response.ok) {
const error = await response.text();
throw new Error(`LLM API call failed: ${response.status} - ${error}`);
}
return response.json();
}
return response.json();
}
parseResponse(response) {
try {
const content = response.content[0].text;
const jsonStart = content.indexOf('{');
const jsonEnd = content.lastIndexOf('}');
if (jsonStart === -1 || jsonEnd === -1) {
throw new Error('No JSON found in response');
}
parseResponse(response) {
try {
const content = response.content[0].text;
const jsonStart = content.indexOf('{');
const jsonEnd = content.lastIndexOf('}');
const jsonString = content.substring(jsonStart, jsonEnd + 1);
return JSON.parse(jsonString);
} catch (error) {
this.logger.error(`Failed to parse LLM response: ${error.message}`);
return null;
}
}
if (jsonStart === -1 || jsonEnd === -1) {
throw new Error('No JSON found in response');
}
calculateCost(usage) {
const modelCosts = {
'claude-3-7-sonnet-20250219': {
input: 3.00, // per 1M tokens
output: 15.00 // per 1M tokens
}
};
const jsonString = content.substring(jsonStart, jsonEnd + 1);
return JSON.parse(jsonString);
} catch (error) {
this.logger.error(`Failed to parse LLM response: ${error.message}`);
return null;
}
}
const costs = modelCosts[this.config.llmAnalysis.model] || { input: 0, output: 0 };
const inputCost = (usage.input_tokens / 1000000) * costs.input;
const outputCost = (usage.output_tokens / 1000000) * costs.output;
return inputCost + outputCost;
}
calculateCost(usage) {
const modelCosts = {
'claude-3-7-sonnet-20250219': {
input: 3.0, // per 1M tokens
output: 15.0 // per 1M tokens
}
};
formatReport(analysis) {
if (!analysis) return null;
const costs = modelCosts[this.config.llmAnalysis.model] || {
input: 0,
output: 0
};
const inputCost = (usage.input_tokens / 1000000) * costs.input;
const outputCost = (usage.output_tokens / 1000000) * costs.output;
const report = {
title: 'TASKMASTER E2E Test Analysis Report',
timestamp: new Date().toISOString(),
status: analysis.overall_status,
summary: analysis.llm_summary_points,
verifiedSteps: analysis.verified_steps,
providerComparison: analysis.provider_add_task_comparison,
issues: analysis.detected_issues
};
return inputCost + outputCost;
}
return report;
}
}
formatReport(analysis) {
if (!analysis) return null;
const report = {
title: 'TASKMASTER E2E Test Analysis Report',
timestamp: new Date().toISOString(),
status: analysis.overall_status,
summary: analysis.llm_summary_points,
verifiedSteps: analysis.verified_steps,
providerComparison: analysis.provider_add_task_comparison,
issues: analysis.detected_issues
};
return report;
}
}

217
tests/e2e/utils/logger.js
View File

@@ -3,122 +3,129 @@ import { join } from 'path';
import chalk from 'chalk';
export class TestLogger {
constructor(logDir, testRunId) {
this.logDir = logDir;
this.testRunId = testRunId;
this.startTime = Date.now();
this.stepCount = 0;
this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
this.logBuffer = [];
this.totalCost = 0;
constructor(logDir, testRunId) {
this.logDir = logDir;
this.testRunId = testRunId;
this.startTime = Date.now();
this.stepCount = 0;
this.logFile = join(logDir, `e2e_run_${testRunId}.log`);
this.logBuffer = [];
this.totalCost = 0;
// Ensure log directory exists
if (!existsSync(logDir)) {
mkdirSync(logDir, { recursive: true });
}
}
// Ensure log directory exists
if (!existsSync(logDir)) {
mkdirSync(logDir, { recursive: true });
}
}
formatDuration(milliseconds) {
const totalSeconds = Math.floor(milliseconds / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
}
formatDuration(milliseconds) {
const totalSeconds = Math.floor(milliseconds / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}m${seconds.toString().padStart(2, '0')}s`;
}
getElapsedTime() {
return this.formatDuration(Date.now() - this.startTime);
}
getElapsedTime() {
return this.formatDuration(Date.now() - this.startTime);
}
formatLogEntry(level, message) {
const timestamp = new Date().toISOString();
const elapsed = this.getElapsedTime();
return `[${level}] [${elapsed}] ${timestamp} ${message}`;
}
formatLogEntry(level, message) {
const timestamp = new Date().toISOString();
const elapsed = this.getElapsedTime();
return `[${level}] [${elapsed}] ${timestamp} ${message}`;
}
log(level, message, options = {}) {
const formattedMessage = this.formatLogEntry(level, message);
// Add to buffer
this.logBuffer.push(formattedMessage);
// Console output with colors
let coloredMessage = formattedMessage;
switch (level) {
case 'INFO':
coloredMessage = chalk.blue(formattedMessage);
break;
case 'SUCCESS':
coloredMessage = chalk.green(formattedMessage);
break;
case 'ERROR':
coloredMessage = chalk.red(formattedMessage);
break;
case 'WARNING':
coloredMessage = chalk.yellow(formattedMessage);
break;
}
console.log(coloredMessage);
// Write to file if immediate flush requested
if (options.flush) {
this.flush();
}
}
log(level, message, options = {}) {
const formattedMessage = this.formatLogEntry(level, message);
info(message) {
this.log('INFO', message);
}
// Add to buffer
this.logBuffer.push(formattedMessage);
success(message) {
this.log('SUCCESS', message);
}
// Console output with colors
let coloredMessage = formattedMessage;
switch (level) {
case 'INFO':
coloredMessage = chalk.blue(formattedMessage);
break;
case 'SUCCESS':
coloredMessage = chalk.green(formattedMessage);
break;
case 'ERROR':
coloredMessage = chalk.red(formattedMessage);
break;
case 'WARNING':
coloredMessage = chalk.yellow(formattedMessage);
break;
}
error(message) {
this.log('ERROR', message);
}
console.log(coloredMessage);
warning(message) {
this.log('WARNING', message);
}
// Write to file if immediate flush requested
if (options.flush) {
this.flush();
}
}
step(message) {
this.stepCount++;
const separator = '='.repeat(45);
this.log('STEP', `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`);
}
info(message) {
this.log('INFO', message);
}
addCost(cost) {
if (typeof cost === 'number' && !isNaN(cost)) {
this.totalCost += cost;
}
}
success(message) {
this.log('SUCCESS', message);
}
extractAndAddCost(output) {
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
let match;
while ((match = costRegex.exec(output)) !== null) {
const cost = parseFloat(match[1]);
this.addCost(cost);
}
}
error(message) {
this.log('ERROR', message);
}
flush() {
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
}
warning(message) {
this.log('WARNING', message);
}
getSummary() {
const duration = this.formatDuration(Date.now() - this.startTime);
const successCount = this.logBuffer.filter(line => line.includes('[SUCCESS]')).length;
const errorCount = this.logBuffer.filter(line => line.includes('[ERROR]')).length;
return {
duration,
totalSteps: this.stepCount,
successCount,
errorCount,
totalCost: this.totalCost.toFixed(6),
logFile: this.logFile
};
}
}
step(message) {
this.stepCount++;
const separator = '='.repeat(45);
this.log(
'STEP',
`\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`
);
}
addCost(cost) {
if (typeof cost === 'number' && !isNaN(cost)) {
this.totalCost += cost;
}
}
extractAndAddCost(output) {
const costRegex = /Est\. Cost: \$(\d+\.\d+)/g;
let match;
while ((match = costRegex.exec(output)) !== null) {
const cost = parseFloat(match[1]);
this.addCost(cost);
}
}
flush() {
writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8');
}
getSummary() {
const duration = this.formatDuration(Date.now() - this.startTime);
const successCount = this.logBuffer.filter((line) =>
line.includes('[SUCCESS]')
).length;
const errorCount = this.logBuffer.filter((line) =>
line.includes('[ERROR]')
).length;
return {
duration,
totalSteps: this.stepCount,
successCount,
errorCount,
totalCost: this.totalCost.toFixed(6),
logFile: this.logFile
};
}
}

337
tests/e2e/utils/test-helpers.js
View File

@@ -3,185 +3,190 @@ import { readFileSync, existsSync, copyFileSync } from 'fs';
import { join } from 'path';
export class TestHelpers {
constructor(logger) {
this.logger = logger;
}
constructor(logger) {
this.logger = logger;
}
/**
* Execute a command and return output
* @param {string} command - Command to execute
* @param {string[]} args - Command arguments
* @param {Object} options - Execution options
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
*/
async executeCommand(command, args = [], options = {}) {
return new Promise((resolve) => {
const spawnOptions = {
cwd: options.cwd || process.cwd(),
env: { ...process.env, ...options.env },
shell: true
};
/**
* Execute a command and return output
* @param {string} command - Command to execute
* @param {string[]} args - Command arguments
* @param {Object} options - Execution options
* @returns {Promise<{stdout: string, stderr: string, exitCode: number}>}
*/
async executeCommand(command, args = [], options = {}) {
return new Promise((resolve) => {
const spawnOptions = {
cwd: options.cwd || process.cwd(),
env: { ...process.env, ...options.env },
shell: true
};
// When using shell: true, pass the full command as a single string
const fullCommand = args.length > 0 ? `${command} ${args.join(' ')}` : command;
const child = spawn(fullCommand, [], spawnOptions);
let stdout = '';
let stderr = '';
// When using shell: true, pass the full command as a single string
const fullCommand =
args.length > 0 ? `${command} ${args.join(' ')}` : command;
const child = spawn(fullCommand, [], spawnOptions);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (exitCode) => {
const output = stdout + stderr;
// Extract and log costs
this.logger.extractAndAddCost(output);
resolve({ stdout, stderr, exitCode });
});
child.on('close', (exitCode) => {
const output = stdout + stderr;
// Handle timeout
if (options.timeout) {
setTimeout(() => {
child.kill('SIGTERM');
}, options.timeout);
}
});
}
// Extract and log costs
this.logger.extractAndAddCost(output);
/**
* Execute task-master command
* @param {string} subcommand - Task-master subcommand
* @param {string[]} args - Command arguments
* @param {Object} options - Execution options
*/
async taskMaster(subcommand, args = [], options = {}) {
const fullArgs = [subcommand, ...args];
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
const result = await this.executeCommand('task-master', fullArgs, options);
if (result.exitCode !== 0 && !options.allowFailure) {
this.logger.error(`Command failed with exit code ${result.exitCode}`);
this.logger.error(`stderr: ${result.stderr}`);
}
return result;
}
resolve({ stdout, stderr, exitCode });
});
/**
* Check if a file exists
*/
fileExists(filePath) {
return existsSync(filePath);
}
// Handle timeout
if (options.timeout) {
setTimeout(() => {
child.kill('SIGTERM');
}, options.timeout);
}
});
}
/**
* Read JSON file
*/
readJson(filePath) {
try {
const content = readFileSync(filePath, 'utf8');
return JSON.parse(content);
} catch (error) {
this.logger.error(`Failed to read JSON file ${filePath}: ${error.message}`);
return null;
}
}
/**
* Execute task-master command
* @param {string} subcommand - Task-master subcommand
* @param {string[]} args - Command arguments
* @param {Object} options - Execution options
*/
async taskMaster(subcommand, args = [], options = {}) {
const fullArgs = [subcommand, ...args];
this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`);
/**
* Copy file
*/
copyFile(source, destination) {
try {
copyFileSync(source, destination);
return true;
} catch (error) {
this.logger.error(`Failed to copy file from ${source} to ${destination}: ${error.message}`);
return false;
}
}
const result = await this.executeCommand('task-master', fullArgs, options);
/**
* Wait for a specified duration
*/
async wait(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
if (result.exitCode !== 0 && !options.allowFailure) {
this.logger.error(`Command failed with exit code ${result.exitCode}`);
this.logger.error(`stderr: ${result.stderr}`);
}
/**
* Verify task exists in tasks.json
*/
verifyTaskExists(tasksFile, taskId, tagName = 'master') {
const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return false;
return tasks[tagName].tasks.some(task => task.id === taskId);
}
return result;
}
/**
* Get task count for a tag
*/
getTaskCount(tasksFile, tagName = 'master') {
const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return 0;
return tasks[tagName].tasks.length;
}
/**
* Check if a file exists
*/
fileExists(filePath) {
return existsSync(filePath);
}
/**
* Extract task ID from command output
*/
extractTaskId(output) {
const patterns = [
/✓ Added new task #(\d+(?:\.\d+)?)/,
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
/Task (\d+(?:\.\d+)?) Created Successfully/
];
for (const pattern of patterns) {
const match = output.match(pattern);
if (match) {
return match[1];
}
}
return null;
}
/**
* Read JSON file
*/
readJson(filePath) {
try {
const content = readFileSync(filePath, 'utf8');
return JSON.parse(content);
} catch (error) {
this.logger.error(
`Failed to read JSON file ${filePath}: ${error.message}`
);
return null;
}
}
/**
* Run multiple async operations in parallel
*/
async runParallel(operations) {
return Promise.all(operations);
}
/**
* Copy file
*/
copyFile(source, destination) {
try {
copyFileSync(source, destination);
return true;
} catch (error) {
this.logger.error(
`Failed to copy file from ${source} to ${destination}: ${error.message}`
);
return false;
}
}
/**
* Run operations with concurrency limit
*/
async runWithConcurrency(operations, limit = 3) {
const results = [];
const executing = [];
for (const operation of operations) {
const promise = operation().then(result => {
executing.splice(executing.indexOf(promise), 1);
return result;
});
results.push(promise);
executing.push(promise);
if (executing.length >= limit) {
await Promise.race(executing);
}
}
return Promise.all(results);
}
}
/**
* Wait for a specified duration
*/
async wait(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
/**
* Verify task exists in tasks.json
*/
verifyTaskExists(tasksFile, taskId, tagName = 'master') {
const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return false;
return tasks[tagName].tasks.some((task) => task.id === taskId);
}
/**
* Get task count for a tag
*/
getTaskCount(tasksFile, tagName = 'master') {
const tasks = this.readJson(tasksFile);
if (!tasks || !tasks[tagName]) return 0;
return tasks[tagName].tasks.length;
}
/**
* Extract task ID from command output
*/
extractTaskId(output) {
const patterns = [
/✓ Added new task #(\d+(?:\.\d+)?)/,
/✅ New task created successfully:.*?(\d+(?:\.\d+)?)/,
/Task (\d+(?:\.\d+)?) Created Successfully/
];
for (const pattern of patterns) {
const match = output.match(pattern);
if (match) {
return match[1];
}
}
return null;
}
/**
* Run multiple async operations in parallel
*/
async runParallel(operations) {
return Promise.all(operations);
}
/**
* Run operations with concurrency limit
*/
async runWithConcurrency(operations, limit = 3) {
const results = [];
const executing = [];
for (const operation of operations) {
const promise = operation().then((result) => {
executing.splice(executing.indexOf(promise), 1);
return result;
});
results.push(promise);
executing.push(promise);
if (executing.length >= limit) {
await Promise.race(executing);
}
}
return Promise.all(results);
}
}