diff --git a/package.json b/package.json index 7af991f2..d8c223e7 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,16 @@ "test:fails": "node --experimental-vm-modules node_modules/.bin/jest --onlyFailures", "test:watch": "node --experimental-vm-modules node_modules/.bin/jest --watch", "test:coverage": "node --experimental-vm-modules node_modules/.bin/jest --coverage", - "test:e2e": "./tests/e2e/run_e2e.sh", - "test:e2e-report": "./tests/e2e/run_e2e.sh --analyze-log", + "test:e2e:bash": "./tests/e2e/run_e2e.sh", + "test:e2e:bash:analyze": "./tests/e2e/run_e2e.sh --analyze-log", + "test:e2e": "node tests/e2e/run-e2e-tests.js", + "test:e2e:parallel": "node tests/e2e/run-e2e-tests.js --parallel", + "test:e2e:sequential": "node tests/e2e/run-e2e-tests.js --sequential", + "test:e2e:analyze": "node tests/e2e/run-e2e-tests.js --analyze-log", + "test:e2e:setup": "node tests/e2e/run-e2e-tests.js --groups setup", + "test:e2e:core": "node tests/e2e/run-e2e-tests.js --groups core", + "test:e2e:providers": "node tests/e2e/run-e2e-tests.js --groups providers", + "test:e2e:advanced": "node tests/e2e/run-e2e-tests.js --groups advanced", "prepare": "chmod +x bin/task-master.js mcp-server/server.js", "changeset": "changeset", "release": "changeset publish", diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..4a3bedc7 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,204 @@ +# Task Master E2E Tests + +This directory contains the modern end-to-end test suite for Task Master AI. The JavaScript implementation provides parallel execution, better error handling, and improved maintainability compared to the legacy bash script. + +## Features + +- **Parallel Execution**: Run test groups concurrently for faster test completion +- **Modular Architecture**: Tests are organized into logical groups (setup, core, providers, advanced) +- **Comprehensive Logging**: Detailed logs with timestamps, cost tracking, and color-coded output +- **LLM Analysis**: Automatic analysis of test results using AI +- **Error Handling**: Robust error handling with categorization and recommendations +- **Flexible Configuration**: Easy to configure test settings and provider configurations + +## Structure + +``` +tests/e2e/ +├── config/ +│ └── test-config.js # Test configuration and settings +├── utils/ +│ ├── logger.js # Test logging utilities +│ ├── test-helpers.js # Common test helper functions +│ ├── llm-analyzer.js # LLM-based log analysis +│ └── error-handler.js # Error handling and reporting +├── tests/ +│ ├── setup.test.js # Setup and initialization tests +│ ├── core.test.js # Core task management tests +│ ├── providers.test.js # Multi-provider tests +│ └── advanced.test.js # Advanced feature tests +├── runners/ +│ ├── parallel-runner.js # Parallel test execution +│ └── test-worker.js # Worker thread for parallel execution +├── run-e2e-tests.js # Main test runner +├── run_e2e.sh # Legacy bash implementation +└── e2e_helpers.sh # Legacy bash helpers +``` + +## Usage + +### Run All Tests (Recommended) + +```bash +# Runs all test groups in the correct order +npm run test:e2e +``` + +### Run Tests Sequentially + +```bash +# Runs all test groups sequentially instead of in parallel +npm run test:e2e:sequential +``` + +### Run Individual Test Groups + +Each test command automatically handles setup if needed, creating a fresh test directory: + +```bash +# Each command creates its own test environment automatically +npm run test:e2e:setup # Setup only (initialize, parse PRD, analyze complexity) +npm run test:e2e:core # Auto-runs setup + core tests (task CRUD, dependencies, status) +npm run test:e2e:providers # Auto-runs setup + provider tests (multi-provider testing) +npm run test:e2e:advanced # Auto-runs setup + advanced tests (tags, subtasks, expand) +``` + +**Note**: Each command creates a fresh test directory, so running individual tests will not share state. This ensures test isolation but means each run will parse the PRD and set up from scratch. + +### Run Multiple Groups + +```bash +# Specify multiple groups to run together +node tests/e2e/run-e2e-tests.js --groups core,providers + +# This automatically runs setup first if needed +node tests/e2e/run-e2e-tests.js --groups providers,advanced +``` + +### Run Tests Against Existing Directory + +If you want to reuse a test directory from a previous run: + +```bash +# First, find your test directory from a previous run: +ls tests/e2e/_runs/ + +# Then run specific tests against that directory: +node tests/e2e/run-e2e-tests.js --groups core --test-dir tests/e2e/_runs/run_2025-07-03_094800611 +``` + +### Analyze Existing Log +```bash +npm run test:e2e:analyze + +# Or analyze specific log file +node tests/e2e/run-e2e-tests.js --analyze-log path/to/log.log +``` + +### Skip Verification Tests +```bash +node tests/e2e/run-e2e-tests.js --skip-verification +``` + +### Run Legacy Bash Tests +```bash +npm run test:e2e:bash +``` + +## Test Groups + +### Setup (`setup`) +- NPM global linking +- Project initialization +- PRD parsing +- Complexity analysis + +### Core (`core`) +- Task CRUD operations +- Dependency management +- Status management +- Subtask operations + +### Providers (`providers`) +- Multi-provider add-task testing +- Provider comparison +- Model switching +- Error handling per provider + +### Advanced (`advanced`) +- Tag management +- Model configuration +- Task expansion +- File generation + +## Configuration + +Edit `config/test-config.js` to customize: + +- Test paths and directories +- Provider configurations +- Test prompts +- Parallel execution settings +- LLM analysis settings + +## Output + +- **Log Files**: Saved to `tests/e2e/log/` with timestamp +- **Test Artifacts**: Created in `tests/e2e/_runs/run_TIMESTAMP/` +- **Console Output**: Color-coded with progress indicators +- **Cost Tracking**: Automatic tracking of AI API costs + +## Requirements + +- Node.js >= 18.0.0 +- Dependencies: chalk, boxen, dotenv, node-fetch +- System utilities: jq, bc +- Valid API keys in `.env` file + +## Comparison with Bash Tests + +| Feature | Bash Script | JavaScript | +|---------|------------|------------| +| Parallel Execution | ❌ | ✅ | +| Error Categorization | Basic | Advanced | +| Test Isolation | Limited | Full | +| Performance | Slower | Faster | +| Debugging | Harder | Easier | +| Cross-platform | Limited | Better | + +## Troubleshooting + +1. **Missing Dependencies**: Install system utilities with `brew install jq bc` (macOS) or `apt-get install jq bc` (Linux) +2. **API Errors**: Check `.env` file for valid API keys +3. **Permission Errors**: Ensure proper file permissions +4. **Timeout Issues**: Adjust timeout in config file + +## Development + +To add new tests: + +1. Create a new test file in `tests/` directory +2. Export a default async function that accepts (logger, helpers, context) +3. Return a results object with status and errors +4. Add the test to appropriate group in `test-config.js` + +Example test structure: +```javascript +export default async function myTest(logger, helpers, context) { + const results = { + status: 'passed', + errors: [] + }; + + try { + logger.step('Running my test'); + // Test implementation + logger.success('Test passed'); + } catch (error) { + results.status = 'failed'; + results.errors.push(error.message); + } + + return results; +} +``` \ No newline at end of file diff --git a/tests/e2e/config/test-config.js b/tests/e2e/config/test-config.js new file mode 100644 index 00000000..7d68551d --- /dev/null +++ b/tests/e2e/config/test-config.js @@ -0,0 +1,65 @@ +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { config as dotenvConfig } from 'dotenv'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Load environment variables +const projectRoot = join(__dirname, '../../..'); +dotenvConfig({ path: join(projectRoot, '.env') }); + +export const testConfig = { + // Paths + paths: { + projectRoot, + sourceDir: projectRoot, + baseTestDir: join(projectRoot, 'tests/e2e/_runs'), + logDir: join(projectRoot, 'tests/e2e/log'), + samplePrdSource: join(projectRoot, 'tests/fixtures/sample-prd.txt'), + mainEnvFile: join(projectRoot, '.env'), + supportedModelsFile: join(projectRoot, 'scripts/modules/supported-models.json') + }, + + // Test settings + settings: { + runVerificationTest: true, + parallelTestGroups: 4, // Number of parallel test groups + timeout: 600000, // 10 minutes default timeout + retryAttempts: 2 + }, + + // Provider test configuration + providers: [ + { name: 'anthropic', model: 'claude-3-7-sonnet-20250219', flags: [] }, + { name: 'openai', model: 'gpt-4o', flags: [] }, + { name: 'google', model: 'gemini-2.5-pro-preview-05-06', flags: [] }, + { name: 'perplexity', model: 'sonar-pro', flags: [] }, + { name: 'xai', model: 'grok-3', flags: [] }, + { name: 'openrouter', model: 'anthropic/claude-3.7-sonnet', flags: [] } + ], + + // Test prompts + prompts: { + addTask: 'Create a task to implement user authentication using OAuth 2.0 with Google as the provider. Include steps for registering the app, handling the callback, and storing user sessions.', + updateTask: 'Update backend server setup: Ensure CORS is configured to allow requests from the frontend origin.', + updateFromTask: 'Refactor the backend storage module to use a simple JSON file (storage.json) instead of an in-memory object for persistence. Update relevant tasks.', + updateSubtask: 'Implementation note: Remember to handle potential API errors and display a user-friendly message.' + }, + + // LLM Analysis settings + llmAnalysis: { + enabled: true, + model: 'claude-3-7-sonnet-20250219', + provider: 'anthropic', + maxTokens: 3072 + } +}; + +// Export test groups for parallel execution +export const testGroups = { + setup: ['setup'], + core: ['core'], + providers: ['providers'], + advanced: ['advanced'] +}; \ No newline at end of file diff --git a/tests/e2e/run-e2e-tests.js b/tests/e2e/run-e2e-tests.js new file mode 100755 index 00000000..a4b0dd55 --- /dev/null +++ b/tests/e2e/run-e2e-tests.js @@ -0,0 +1,392 @@ +#!/usr/bin/env node + +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { existsSync } from 'fs'; +import process from 'process'; +import chalk from 'chalk'; +import boxen from 'boxen'; +import { TestLogger } from './utils/logger.js'; +import { TestHelpers } from './utils/test-helpers.js'; +import { LLMAnalyzer } from './utils/llm-analyzer.js'; +import { testConfig, testGroups } from './config/test-config.js'; +import { + ParallelTestRunner, + SequentialTestRunner +} from './runners/parallel-runner.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Parse command line arguments +function parseArgs() { + const args = process.argv.slice(2); + const options = { + skipVerification: false, + analyzeLog: false, + logFile: null, + parallel: true, + groups: null, + testDir: null + }; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case '--skip-verification': + options.skipVerification = true; + break; + case '--analyze-log': + options.analyzeLog = true; + if (args[i + 1] && !args[i + 1].startsWith('--')) { + options.logFile = args[i + 1]; + i++; + } + break; + case '--sequential': + options.parallel = false; + break; + case '--groups': + if (args[i + 1] && !args[i + 1].startsWith('--')) { + options.groups = args[i + 1].split(','); + i++; + } + break; + case '--test-dir': + if (args[i + 1] && !args[i + 1].startsWith('--')) { + options.testDir = args[i + 1]; + i++; + } + break; + case '--help': + showHelp(); + process.exit(0); + } + } + + return options; +} + +function showHelp() { + console.log( + boxen( + `Task Master E2E Test Runner + +Usage: node run-e2e-tests.js [options] + +Options: + --skip-verification Skip fallback verification tests + --analyze-log [file] Analyze an existing log file + --sequential Run tests sequentially instead of in parallel + --groups Run only specific test groups + --help Show this help message + +Test Groups: ${Object.keys(testGroups).join(', ')}`, + { padding: 1, margin: 1, borderStyle: 'round' } + ) + ); +} + +// Generate timestamp for test run +function generateTimestamp() { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + + return `${year}${month}${day}_${hours}${minutes}${seconds}`; +} + +// Analyze existing log file +async function analyzeLogFile(logFile, logger) { + console.log(chalk.cyan('\n📊 Running LLM Analysis on log file...\n')); + + const analyzer = new LLMAnalyzer(testConfig, logger); + const analysis = await analyzer.analyzeLog(logFile); + + if (analysis) { + const report = analyzer.formatReport(analysis); + displayAnalysisReport(report); + } else { + console.log(chalk.red('Failed to analyze log file')); + } +} + +// Display analysis report +function displayAnalysisReport(report) { + console.log( + boxen(chalk.cyan.bold(`${report.title}\n${report.timestamp}`), { + padding: 1, + borderStyle: 'double', + borderColor: 'cyan' + }) + ); + + // Status + const statusColor = + report.status === 'Success' + ? chalk.green + : report.status === 'Warning' + ? chalk.yellow + : chalk.red; + console.log( + boxen(`Overall Status: ${statusColor.bold(report.status)}`, { + padding: { left: 1, right: 1 }, + borderColor: 'blue' + }) + ); + + // Summary points + if (report.summary && report.summary.length > 0) { + console.log(chalk.blue.bold('\n📋 Summary Points:')); + report.summary.forEach((point) => { + console.log(chalk.white(` - ${point}`)); + }); + } + + // Provider comparison + if (report.providerComparison) { + console.log(chalk.magenta.bold('\n🔄 Provider Comparison:')); + const comp = report.providerComparison; + + if (comp.provider_results) { + Object.entries(comp.provider_results).forEach(([provider, result]) => { + const status = + result.status === 'Success' ? chalk.green('✅') : chalk.red('❌'); + console.log(` ${status} ${provider}: ${result.notes || 'N/A'}`); + }); + } + } + + // Issues + if (report.issues && report.issues.length > 0) { + console.log(chalk.red.bold('\n🚨 Detected Issues:')); + report.issues.forEach((issue, index) => { + const severity = + issue.severity === 'Error' + ? chalk.red('❌') + : issue.severity === 'Warning' + ? chalk.yellow('⚠️') + : 'ℹ️'; + console.log( + boxen( + `${severity} Issue ${index + 1}: [${issue.severity}]\n${issue.description}`, + { padding: 1, margin: { bottom: 1 }, borderColor: 'red' } + ) + ); + }); + } +} + +// Main test execution +async function runTests(options) { + const timestamp = generateTimestamp(); + const logger = new TestLogger(testConfig.paths.logDir, timestamp); + const helpers = new TestHelpers(logger); + + console.log( + boxen( + chalk.cyan.bold( + `Task Master E2E Test Suite\n${new Date().toISOString()}` + ), + { padding: 1, borderStyle: 'double', borderColor: 'cyan' } + ) + ); + + logger.info('Starting E2E test suite'); + logger.info(`Test configuration: ${JSON.stringify(options)}`); + + // Update config based on options + if (options.skipVerification) { + testConfig.settings.runVerificationTest = false; + } + + let results; + let testDir; + + try { + // Check dependencies + logger.step('Checking dependencies'); + const deps = ['jq', 'bc']; + for (const dep of deps) { + const { exitCode } = await helpers.executeCommand('which', [dep]); + if (exitCode !== 0) { + throw new Error( + `Required dependency '${dep}' not found. Please install it.` + ); + } + } + logger.success('All dependencies found'); + + // Determine which test groups to run + const groupsToRun = options.groups || Object.keys(testGroups); + const testsToRun = {}; + + groupsToRun.forEach((group) => { + if (testGroups[group]) { + testsToRun[group] = testGroups[group]; + } else { + logger.warning(`Unknown test group: ${group}`); + } + }); + + if (Object.keys(testsToRun).length === 0) { + throw new Error('No valid test groups to run'); + } + + // Check if we need to run setup (either explicitly requested or needed for other tests) + const needsSetup = testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0); + + if (needsSetup) { + // Always run setup if we need a test directory + if (!testsToRun.setup) { + logger.info('No test directory available, running setup automatically'); + } + + logger.step('Running setup tests'); + const setupRunner = new SequentialTestRunner(logger, helpers); + const setupResults = await setupRunner.runTests(['setup'], {}); + + if (setupResults.tests.setup && setupResults.tests.setup.testDir) { + testDir = setupResults.tests.setup.testDir; + logger.info(`Test directory created: ${testDir}`); + } else { + throw new Error('Setup failed to create test directory'); + } + + // Remove setup from remaining tests if it was explicitly requested + if (testsToRun.setup) { + delete testsToRun.setup; + } + } + + // Run remaining tests + if (Object.keys(testsToRun).length > 0) { + const context = { testDir, config: testConfig }; + + if (options.parallel) { + logger.info('Running tests in parallel mode'); + const parallelRunner = new ParallelTestRunner(logger); + + try { + results = await parallelRunner.runTestGroups(testsToRun, context); + } finally { + await parallelRunner.terminate(); + } + } else { + logger.info('Running tests in sequential mode'); + const sequentialRunner = new SequentialTestRunner(logger, helpers); + + // Flatten test groups for sequential execution + const allTests = Object.values(testsToRun).flat(); + results = await sequentialRunner.runTests(allTests, context); + } + } + + // Final summary + logger.flush(); + const summary = logger.getSummary(); + + displayTestSummary(summary, results); + + // Run LLM analysis if enabled + if (testConfig.llmAnalysis.enabled && !options.skipVerification) { + await analyzeLogFile(summary.logFile, logger); + } + + // Exit with appropriate code + const exitCode = results && results.overall === 'passed' ? 0 : 1; + process.exit(exitCode); + } catch (error) { + logger.error(`Fatal error: ${error.message}`); + logger.flush(); + + console.log(chalk.red.bold('\n❌ Test execution failed')); + console.log(chalk.red(error.stack)); + + process.exit(1); + } +} + +// Display test summary +function displayTestSummary(summary, results) { + console.log( + boxen(chalk.cyan.bold('E2E Test Summary'), { + padding: 1, + margin: { top: 1 }, + borderStyle: 'round', + borderColor: 'cyan' + }) + ); + + console.log(chalk.white(`📁 Log File: ${summary.logFile}`)); + console.log(chalk.white(`⏱️ Duration: ${summary.duration}`)); + console.log(chalk.white(`📊 Total Steps: ${summary.totalSteps}`)); + console.log(chalk.green(`✅ Successes: ${summary.successCount}`)); + + if (summary.errorCount > 0) { + console.log(chalk.red(`❌ Errors: ${summary.errorCount}`)); + } + + console.log(chalk.yellow(`💰 Total Cost: $${summary.totalCost} USD`)); + + if (results) { + const status = + results.overall === 'passed' + ? chalk.green.bold('✅ PASSED') + : chalk.red.bold('❌ FAILED'); + + console.log( + boxen(`Overall Result: ${status}`, { + padding: 1, + margin: { top: 1 }, + borderColor: results.overall === 'passed' ? 'green' : 'red' + }) + ); + } +} + +// Main entry point +async function main() { + const options = parseArgs(); + + if (options.analyzeLog) { + // Analysis mode + const logFile = options.logFile || (await findLatestLog()); + const logger = new TestLogger(testConfig.paths.logDir, 'analysis'); + + if (!existsSync(logFile)) { + console.error(chalk.red(`Log file not found: ${logFile}`)); + process.exit(1); + } + + await analyzeLogFile(logFile, logger); + } else { + // Test execution mode + await runTests(options); + } +} + +// Find the latest log file +async function findLatestLog() { + const { readdir } = await import('fs/promises'); + const files = await readdir(testConfig.paths.logDir); + const logFiles = files + .filter((f) => f.startsWith('e2e_run_') && f.endsWith('.log')) + .sort() + .reverse(); + + if (logFiles.length === 0) { + throw new Error('No log files found'); + } + + return join(testConfig.paths.logDir, logFiles[0]); +} + +// Run the main function +main().catch((error) => { + console.error(chalk.red('Unexpected error:'), error); + process.exit(1); +}); diff --git a/tests/e2e/runners/parallel-runner.js b/tests/e2e/runners/parallel-runner.js new file mode 100644 index 00000000..e621aba9 --- /dev/null +++ b/tests/e2e/runners/parallel-runner.js @@ -0,0 +1,211 @@ +import { Worker } from 'worker_threads'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { EventEmitter } from 'events'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +export class ParallelTestRunner extends EventEmitter { + constructor(logger) { + super(); + this.logger = logger; + this.workers = []; + this.results = {}; + } + + /** + * Run test groups in parallel + * @param {Object} testGroups - Groups of tests to run + * @param {Object} sharedContext - Shared context for all tests + * @returns {Promise} Combined results from all test groups + */ + async runTestGroups(testGroups, sharedContext) { + const groupNames = Object.keys(testGroups); + const workerPromises = []; + + this.logger.info(`Starting parallel execution of ${groupNames.length} test groups`); + + for (const groupName of groupNames) { + const workerPromise = this.runTestGroup(groupName, testGroups[groupName], sharedContext); + workerPromises.push(workerPromise); + } + + // Wait for all workers to complete + const results = await Promise.allSettled(workerPromises); + + // Process results + const combinedResults = { + overall: 'passed', + groups: {}, + summary: { + totalGroups: groupNames.length, + passedGroups: 0, + failedGroups: 0, + errors: [] + } + }; + + results.forEach((result, index) => { + const groupName = groupNames[index]; + + if (result.status === 'fulfilled') { + combinedResults.groups[groupName] = result.value; + if (result.value.status === 'passed') { + combinedResults.summary.passedGroups++; + } else { + combinedResults.summary.failedGroups++; + combinedResults.overall = 'failed'; + } + } else { + combinedResults.groups[groupName] = { + status: 'failed', + error: result.reason.message || 'Unknown error' + }; + combinedResults.summary.failedGroups++; + combinedResults.summary.errors.push({ + group: groupName, + error: result.reason.message + }); + combinedResults.overall = 'failed'; + } + }); + + return combinedResults; + } + + /** + * Run a single test group in a worker thread + */ + async runTestGroup(groupName, testModules, sharedContext) { + return new Promise((resolve, reject) => { + const workerPath = join(__dirname, 'test-worker.js'); + + const worker = new Worker(workerPath, { + workerData: { + groupName, + testModules, + sharedContext, + logDir: this.logger.logDir, + testRunId: this.logger.testRunId + } + }); + + this.workers.push(worker); + + // Handle messages from worker + worker.on('message', (message) => { + if (message.type === 'log') { + const level = message.level.toLowerCase(); + if (typeof this.logger[level] === 'function') { + this.logger[level](message.message); + } else { + // Fallback to info if the level doesn't exist + this.logger.info(message.message); + } + } else if (message.type === 'step') { + this.logger.step(message.message); + } else if (message.type === 'cost') { + this.logger.addCost(message.cost); + } else if (message.type === 'results') { + this.results[groupName] = message.results; + } + }); + + // Handle worker completion + worker.on('exit', (code) => { + this.workers = this.workers.filter(w => w !== worker); + + if (code === 0) { + resolve(this.results[groupName] || { status: 'passed', group: groupName }); + } else { + reject(new Error(`Worker for group ${groupName} exited with code ${code}`)); + } + }); + + // Handle worker errors + worker.on('error', (error) => { + this.workers = this.workers.filter(w => w !== worker); + reject(error); + }); + + }); + } + + /** + * Terminate all running workers + */ + async terminate() { + const terminationPromises = this.workers.map(worker => + worker.terminate().catch(err => + this.logger.warning(`Failed to terminate worker: ${err.message}`) + ) + ); + + await Promise.all(terminationPromises); + this.workers = []; + } +} + +/** + * Sequential test runner for comparison or fallback + */ +export class SequentialTestRunner { + constructor(logger, helpers) { + this.logger = logger; + this.helpers = helpers; + } + + /** + * Run tests sequentially + */ + async runTests(testModules, context) { + const results = { + overall: 'passed', + tests: {}, + summary: { + totalTests: testModules.length, + passedTests: 0, + failedTests: 0, + errors: [] + } + }; + + for (const testModule of testModules) { + try { + this.logger.step(`Running ${testModule} tests`); + + // Dynamic import of test module + const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`); + const { default: testFn } = await import(testPath); + + // Run the test + const testResults = await testFn(this.logger, this.helpers, context); + + results.tests[testModule] = testResults; + + if (testResults.status === 'passed') { + results.summary.passedTests++; + } else { + results.summary.failedTests++; + results.overall = 'failed'; + } + + } catch (error) { + this.logger.error(`Failed to run ${testModule}: ${error.message}`); + results.tests[testModule] = { + status: 'failed', + error: error.message + }; + results.summary.failedTests++; + results.summary.errors.push({ + test: testModule, + error: error.message + }); + results.overall = 'failed'; + } + } + + return results; + } +} \ No newline at end of file diff --git a/tests/e2e/runners/test-worker.js b/tests/e2e/runners/test-worker.js new file mode 100644 index 00000000..89536b08 --- /dev/null +++ b/tests/e2e/runners/test-worker.js @@ -0,0 +1,132 @@ +import { parentPort, workerData } from 'worker_threads'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { TestLogger } from '../utils/logger.js'; +import { TestHelpers } from '../utils/test-helpers.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Worker logger that sends messages to parent +class WorkerLogger extends TestLogger { + constructor(logDir, testRunId, groupName) { + super(logDir, `${testRunId}_${groupName}`); + this.groupName = groupName; + } + + log(level, message, options = {}) { + super.log(level, message, options); + + // Send log to parent + parentPort.postMessage({ + type: 'log', + level: level.toLowerCase(), + message: `[${this.groupName}] ${message}` + }); + } + + step(message) { + super.step(message); + + parentPort.postMessage({ + type: 'step', + message: `[${this.groupName}] ${message}` + }); + } + + addCost(cost) { + super.addCost(cost); + + parentPort.postMessage({ + type: 'cost', + cost + }); + } +} + +// Main worker execution +async function runTestGroup() { + const { groupName, testModules, sharedContext, logDir, testRunId } = workerData; + + const logger = new WorkerLogger(logDir, testRunId, groupName); + const helpers = new TestHelpers(logger); + + logger.info(`Worker started for test group: ${groupName}`); + + const results = { + group: groupName, + status: 'passed', + tests: {}, + errors: [], + startTime: Date.now() + }; + + try { + // Run each test module in the group + for (const testModule of testModules) { + try { + logger.info(`Running test: ${testModule}`); + + // Dynamic import of test module + const testPath = join(dirname(__dirname), 'tests', `${testModule}.test.js`); + const { default: testFn } = await import(testPath); + + // Run the test with shared context + const testResults = await testFn(logger, helpers, sharedContext); + + results.tests[testModule] = testResults; + + if (testResults.status !== 'passed') { + results.status = 'failed'; + if (testResults.errors) { + results.errors.push(...testResults.errors); + } + } + + } catch (error) { + logger.error(`Test ${testModule} failed: ${error.message}`); + results.tests[testModule] = { + status: 'failed', + error: error.message, + stack: error.stack + }; + results.status = 'failed'; + results.errors.push({ + test: testModule, + error: error.message + }); + } + } + + } catch (error) { + logger.error(`Worker error: ${error.message}`); + results.status = 'failed'; + results.errors.push({ + group: groupName, + error: error.message, + stack: error.stack + }); + } + + results.endTime = Date.now(); + results.duration = results.endTime - results.startTime; + + // Flush logs and get summary + logger.flush(); + const summary = logger.getSummary(); + results.summary = summary; + + // Send results to parent + parentPort.postMessage({ + type: 'results', + results + }); + + logger.info(`Worker completed for test group: ${groupName}`); +} + +// Run the test group +runTestGroup().catch(error => { + console.error('Worker fatal error:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/tests/e2e/tests/advanced.test.js b/tests/e2e/tests/advanced.test.js new file mode 100644 index 00000000..446ee1ca --- /dev/null +++ b/tests/e2e/tests/advanced.test.js @@ -0,0 +1,316 @@ +export default async function testAdvanced(logger, helpers, context) { + const { testDir } = context; + logger.info('Starting advanced features tests...'); + const results = []; + + try { + // Test Tag Context Management + logger.info('Testing tag context management...'); + + // Create new tag contexts + const tag1Result = await helpers.taskMaster('add-tag', ['feature-auth', '--description', 'Authentication feature'], { cwd: testDir }); + results.push({ + test: 'Create tag context - feature-auth', + passed: tag1Result.exitCode === 0, + output: tag1Result.stdout + }); + + const tag2Result = await helpers.taskMaster('add-tag', ['feature-api', '--description', 'API development'], { cwd: testDir }); + results.push({ + test: 'Create tag context - feature-api', + passed: tag2Result.exitCode === 0, + output: tag2Result.stdout + }); + + // Add task to feature-auth tag + const task1Result = await helpers.taskMaster('add-task', ['--tag=feature-auth', '--prompt', 'Implement user authentication'], { cwd: testDir }); + results.push({ + test: 'Add task to feature-auth tag', + passed: task1Result.exitCode === 0, + output: task1Result.stdout + }); + + // Add task to feature-api tag + const task2Result = await helpers.taskMaster('add-task', ['--tag=feature-api', '--prompt', 'Create REST API endpoints'], { cwd: testDir }); + results.push({ + test: 'Add task to feature-api tag', + passed: task2Result.exitCode === 0, + output: task2Result.stdout + }); + + // List all tag contexts + const listTagsResult = await helpers.taskMaster('tags', [], { cwd: testDir }); + results.push({ + test: 'List all tag contexts', + passed: listTagsResult.exitCode === 0 && + listTagsResult.stdout.includes('feature-auth') && + listTagsResult.stdout.includes('feature-api'), + output: listTagsResult.stdout + }); + + // List tasks in feature-auth tag + const taggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-auth'], { cwd: testDir }); + results.push({ + test: 'List tasks in feature-auth tag', + passed: taggedTasksResult.exitCode === 0 && + taggedTasksResult.stdout.includes('Implement user authentication'), + output: taggedTasksResult.stdout + }); + + // Test Model Configuration + logger.info('Testing model configuration...'); + + // Set main model + const setMainModelResult = await helpers.taskMaster('models', ['--set-main', 'gpt-4'], { cwd: testDir }); + results.push({ + test: 'Set main model', + passed: setMainModelResult.exitCode === 0, + output: setMainModelResult.stdout + }); + + // Set research model + const setResearchModelResult = await helpers.taskMaster('models', ['--set-research', 'claude-3-sonnet'], { cwd: testDir }); + results.push({ + test: 'Set research model', + passed: setResearchModelResult.exitCode === 0, + output: setResearchModelResult.stdout + }); + + // Set fallback model + const setFallbackModelResult = await helpers.taskMaster('models', ['--set-fallback', 'gpt-3.5-turbo'], { cwd: testDir }); + results.push({ + test: 'Set fallback model', + passed: setFallbackModelResult.exitCode === 0, + output: setFallbackModelResult.stdout + }); + + // Verify model configuration + const showModelsResult = await helpers.taskMaster('models', [], { cwd: testDir }); + results.push({ + test: 'Show model configuration', + passed: showModelsResult.exitCode === 0 && + showModelsResult.stdout.includes('gpt-4') && + showModelsResult.stdout.includes('claude-3-sonnet') && + showModelsResult.stdout.includes('gpt-3.5-turbo'), + output: showModelsResult.stdout + }); + + // Test Task Expansion + logger.info('Testing task expansion...'); + + // Add task for expansion + const expandTaskResult = await helpers.taskMaster('add-task', ['--prompt', 'Build REST API'], { cwd: testDir }); + const expandTaskMatch = expandTaskResult.stdout.match(/#(\d+)/); + const expandTaskId = expandTaskMatch ? expandTaskMatch[1] : null; + + results.push({ + test: 'Add task for expansion', + passed: expandTaskResult.exitCode === 0 && expandTaskId !== null, + output: expandTaskResult.stdout + }); + + if (expandTaskId) { + // Single task expansion + const expandResult = await helpers.taskMaster('expand', [expandTaskId], { cwd: testDir }); + results.push({ + test: 'Expand single task', + passed: expandResult.exitCode === 0 && expandResult.stdout.includes('subtasks'), + output: expandResult.stdout + }); + + // Verify expand worked + const afterExpandResult = await helpers.taskMaster('show', [expandTaskId], { cwd: testDir }); + results.push({ + test: 'Verify task expansion', + passed: afterExpandResult.exitCode === 0 && afterExpandResult.stdout.includes('subtasks'), + output: afterExpandResult.stdout + }); + + // Force expand (re-expand) + const forceExpandResult = await helpers.taskMaster('expand', [expandTaskId, '--force'], { cwd: testDir }); + results.push({ + test: 'Force expand task', + passed: forceExpandResult.exitCode === 0, + output: forceExpandResult.stdout + }); + } + + // Test Subtask Management + logger.info('Testing subtask management...'); + + // Add task for subtask testing + const subtaskParentResult = await helpers.taskMaster('add-task', ['--prompt', 'Create user interface'], { cwd: testDir }); + const parentMatch = subtaskParentResult.stdout.match(/#(\d+)/); + const parentTaskId = parentMatch ? parentMatch[1] : null; + + if (parentTaskId) { + // Add subtasks manually + const addSubtask1Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Design mockups'], { cwd: testDir }); + results.push({ + test: 'Add subtask - Design mockups', + passed: addSubtask1Result.exitCode === 0, + output: addSubtask1Result.stdout + }); + + const addSubtask2Result = await helpers.taskMaster('add-subtask', ['--parent', parentTaskId, '--title', 'Implement components'], { cwd: testDir }); + results.push({ + test: 'Add subtask - Implement components', + passed: addSubtask2Result.exitCode === 0, + output: addSubtask2Result.stdout + }); + + // List subtasks (use show command to see subtasks) + const listSubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); + results.push({ + test: 'List subtasks', + passed: listSubtasksResult.exitCode === 0 && + listSubtasksResult.stdout.includes('Design mockups') && + listSubtasksResult.stdout.includes('Implement components'), + output: listSubtasksResult.stdout + }); + + // Update subtask + const subtaskId = `${parentTaskId}.1`; + const updateSubtaskResult = await helpers.taskMaster('update-subtask', ['--id', subtaskId, '--prompt', 'Create detailed mockups'], { cwd: testDir }); + results.push({ + test: 'Update subtask', + passed: updateSubtaskResult.exitCode === 0, + output: updateSubtaskResult.stdout + }); + + // Remove subtask + const removeSubtaskId = `${parentTaskId}.2`; + const removeSubtaskResult = await helpers.taskMaster('remove-subtask', ['--id', removeSubtaskId], { cwd: testDir }); + results.push({ + test: 'Remove subtask', + passed: removeSubtaskResult.exitCode === 0, + output: removeSubtaskResult.stdout + }); + + // Verify subtask changes + const verifySubtasksResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); + results.push({ + test: 'Verify subtask changes', + passed: verifySubtasksResult.exitCode === 0 && + verifySubtasksResult.stdout.includes('Create detailed mockups') && + !verifySubtasksResult.stdout.includes('Implement components'), + output: verifySubtasksResult.stdout + }); + + // Clear all subtasks + const clearSubtasksResult = await helpers.taskMaster('clear-subtasks', ['--id', parentTaskId], { cwd: testDir }); + results.push({ + test: 'Clear all subtasks', + passed: clearSubtasksResult.exitCode === 0, + output: clearSubtasksResult.stdout + }); + + // Verify subtasks cleared + const verifyClearResult = await helpers.taskMaster('show', [parentTaskId], { cwd: testDir }); + results.push({ + test: 'Verify subtasks cleared', + passed: verifyClearResult.exitCode === 0 && + (!verifyClearResult.stdout.includes('Design mockups') && + !verifyClearResult.stdout.includes('Create detailed mockups')), + output: verifyClearResult.stdout + }); + } + + // Test Expand All + logger.info('Testing expand all...'); + + // Add multiple tasks + await helpers.taskMaster('add-task', ['--prompt', 'Task A for expand all'], { cwd: testDir }); + await helpers.taskMaster('add-task', ['--prompt', 'Task B for expand all'], { cwd: testDir }); + + const expandAllResult = await helpers.taskMaster('expand', ['--all'], { cwd: testDir }); + results.push({ + test: 'Expand all tasks', + passed: expandAllResult.exitCode === 0, + output: expandAllResult.stdout + }); + + // Test Generate Task Files + logger.info('Testing generate task files...'); + + // Generate files for a specific task + if (expandTaskId) { + const generateResult = await helpers.taskMaster('generate', [expandTaskId], { cwd: testDir }); + results.push({ + test: 'Generate task files', + passed: generateResult.exitCode === 0, + output: generateResult.stdout + }); + + // Check if files were created + const taskFilePath = `${testDir}/tasks/task_${expandTaskId}.md`; + const fileExists = helpers.fileExists(taskFilePath); + + results.push({ + test: 'Verify generated task file exists', + passed: fileExists, + output: fileExists ? `Task file created at ${taskFilePath}` : 'Task file not found' + }); + } + + // Test Tag Context Integrity After Operations + logger.info('Testing tag context integrity after operations...'); + + // Verify tag contexts still exist + const finalTagListResult = await helpers.taskMaster('tags', [], { cwd: testDir }); + results.push({ + test: 'Final tag context list verification', + passed: finalTagListResult.exitCode === 0 && + finalTagListResult.stdout.includes('feature-auth') && + finalTagListResult.stdout.includes('feature-api'), + output: finalTagListResult.stdout + }); + + // Verify tasks are still in their respective tag contexts + const finalTaggedTasksResult = await helpers.taskMaster('list', ['--tag=feature-api'], { cwd: testDir }); + results.push({ + test: 'Final tasks in tag context verification', + passed: finalTaggedTasksResult.exitCode === 0 && + finalTaggedTasksResult.stdout.includes('Create REST API endpoints'), + output: finalTaggedTasksResult.stdout + }); + + // Test Additional Advanced Features + logger.info('Testing additional advanced features...'); + + // Test priority task + const priorityTagResult = await helpers.taskMaster('add-task', ['--prompt', 'High priority task', '--priority', 'high'], { cwd: testDir }); + results.push({ + test: 'Add task with high priority', + passed: priorityTagResult.exitCode === 0, + output: priorityTagResult.stdout + }); + + // Test filtering by status + const statusFilterResult = await helpers.taskMaster('list', ['--status', 'pending'], { cwd: testDir }); + results.push({ + test: 'Filter by status', + passed: statusFilterResult.exitCode === 0, + output: statusFilterResult.stdout + }); + + } catch (error) { + logger.error('Error in advanced features tests:', error); + results.push({ + test: 'Advanced features test suite', + passed: false, + error: error.message + }); + } + + const passed = results.filter(r => r.passed).length; + const total = results.length; + + return { + name: 'Advanced Features', + passed, + total, + results, + summary: `Advanced features tests: ${passed}/${total} passed` + }; +}; \ No newline at end of file diff --git a/tests/e2e/tests/core.test.js b/tests/e2e/tests/core.test.js new file mode 100644 index 00000000..fcb8606d --- /dev/null +++ b/tests/e2e/tests/core.test.js @@ -0,0 +1,285 @@ +/** + * Core task operations test module + * Tests all fundamental task management functionality + */ + +export default async function testCoreOperations(logger, helpers, context) { + const { testDir } = context; + const results = { + status: 'passed', + errors: [] + }; + + try { + logger.info('Starting core task operations tests...'); + + // Test 1: List tasks (may have tasks from PRD parsing) + logger.info('\nTest 1: List tasks'); + const listResult1 = await helpers.taskMaster('list', [], { cwd: testDir }); + if (listResult1.exitCode !== 0) { + throw new Error(`List command failed: ${listResult1.stderr}`); + } + // Check for expected output patterns - either empty or with tasks + const hasValidOutput = listResult1.stdout.includes('No tasks found') || + listResult1.stdout.includes('Task List') || + listResult1.stdout.includes('Project Dashboard') || + listResult1.stdout.includes('Listing tasks from'); + if (!hasValidOutput) { + throw new Error('Unexpected list output format'); + } + logger.success('✓ List tasks successful'); + + // Test 2: Add manual task + logger.info('\nTest 2: Add manual task'); + const addResult1 = await helpers.taskMaster('add-task', ['--title', 'Write unit tests', '--description', 'Create comprehensive unit tests for the application'], { cwd: testDir }); + if (addResult1.exitCode !== 0) { + throw new Error(`Failed to add manual task: ${addResult1.stderr}`); + } + const manualTaskId = helpers.extractTaskId(addResult1.stdout); + if (!manualTaskId) { + throw new Error('Failed to extract task ID from add output'); + } + logger.success(`✓ Added manual task with ID: ${manualTaskId}`); + + // Test 3: Add AI task + logger.info('\nTest 3: Add AI task'); + const addResult2 = await helpers.taskMaster('add-task', ['--prompt', 'Implement authentication system'], { cwd: testDir }); + if (addResult2.exitCode !== 0) { + throw new Error(`Failed to add AI task: ${addResult2.stderr}`); + } + const aiTaskId = helpers.extractTaskId(addResult2.stdout); + if (!aiTaskId) { + throw new Error('Failed to extract AI task ID from add output'); + } + logger.success(`✓ Added AI task with ID: ${aiTaskId}`); + + // Test 4: Add another task for dependency testing + logger.info('\nTest 4: Add task for dependency testing'); + const addResult3 = await helpers.taskMaster('add-task', ['--title', 'Create database schema', '--description', 'Design and implement the database schema'], { cwd: testDir }); + if (addResult3.exitCode !== 0) { + throw new Error(`Failed to add database task: ${addResult3.stderr}`); + } + const dbTaskId = helpers.extractTaskId(addResult3.stdout); + if (!dbTaskId) { + throw new Error('Failed to extract database task ID'); + } + logger.success(`✓ Added database task with ID: ${dbTaskId}`); + + // Test 5: List tasks (should show our newly added tasks) + logger.info('\nTest 5: List all tasks'); + const listResult2 = await helpers.taskMaster('list', [], { cwd: testDir }); + if (listResult2.exitCode !== 0) { + throw new Error(`List command failed: ${listResult2.stderr}`); + } + // Check that we can find our task IDs in the output + const hasTask11 = listResult2.stdout.includes('11'); + const hasTask12 = listResult2.stdout.includes('12'); + const hasTask13 = listResult2.stdout.includes('13'); + + if (!hasTask11 || !hasTask12 || !hasTask13) { + throw new Error('Not all task IDs found in list output'); + } + + // Also check for partial matches (list may truncate titles) + const hasOurTasks = listResult2.stdout.includes('Write') || + listResult2.stdout.includes('Create'); + if (hasOurTasks) { + logger.success('✓ List tasks shows our added tasks'); + } else { + logger.warning('Task titles may be truncated in list view'); + } + + // Test 6: Get next task + logger.info('\nTest 6: Get next task'); + const nextResult = await helpers.taskMaster('next', [], { cwd: testDir }); + if (nextResult.exitCode !== 0) { + throw new Error(`Next task command failed: ${nextResult.stderr}`); + } + logger.success('✓ Get next task successful'); + + // Test 7: Show task details + logger.info('\nTest 7: Show task details'); + const showResult = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); + if (showResult.exitCode !== 0) { + throw new Error(`Show task details failed: ${showResult.stderr}`); + } + // Check that the task ID is shown and basic structure is present + if (!showResult.stdout.includes(`Task: #${aiTaskId}`) && !showResult.stdout.includes(`ID: │ ${aiTaskId}`)) { + throw new Error('Task ID not found in show output'); + } + if (!showResult.stdout.includes('Status:') || !showResult.stdout.includes('Priority:')) { + throw new Error('Task details missing expected fields'); + } + logger.success('✓ Show task details successful'); + + // Test 8: Add dependencies + logger.info('\nTest 8: Add dependencies'); + const addDepResult = await helpers.taskMaster('add-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); + if (addDepResult.exitCode !== 0) { + throw new Error(`Failed to add dependency: ${addDepResult.stderr}`); + } + logger.success('✓ Added dependency successfully'); + + // Test 9: Verify dependency was added + logger.info('\nTest 9: Verify dependency'); + const showResult2 = await helpers.taskMaster('show', [aiTaskId], { cwd: testDir }); + if (showResult2.exitCode !== 0) { + throw new Error(`Show task failed: ${showResult2.stderr}`); + } + if (!showResult2.stdout.includes('Dependencies:') || !showResult2.stdout.includes(dbTaskId)) { + throw new Error('Dependency not shown in task details'); + } + logger.success('✓ Dependency verified in task details'); + + // Test 10: Test circular dependency (should fail) + logger.info('\nTest 10: Test circular dependency prevention'); + const circularResult = await helpers.taskMaster('add-dependency', ['--id', dbTaskId, '--depends-on', aiTaskId], { + cwd: testDir, + allowFailure: true + }); + if (circularResult.exitCode === 0) { + throw new Error('Circular dependency was not prevented'); + } + if (!circularResult.stderr.toLowerCase().includes('circular')) { + throw new Error('Expected circular dependency error message'); + } + logger.success('✓ Circular dependency prevented successfully'); + + // Test 11: Test non-existent dependency + logger.info('\nTest 11: Test non-existent dependency'); + const nonExistResult = await helpers.taskMaster('add-dependency', ['--id', '99999', '--depends-on', '88888'], { + cwd: testDir, + allowFailure: true + }); + if (nonExistResult.exitCode === 0) { + throw new Error('Non-existent dependency was incorrectly allowed'); + } + logger.success('✓ Non-existent dependency handled correctly'); + + // Test 12: Remove dependency + logger.info('\nTest 12: Remove dependency'); + const removeDepResult = await helpers.taskMaster('remove-dependency', ['--id', aiTaskId, '--depends-on', dbTaskId], { cwd: testDir }); + if (removeDepResult.exitCode !== 0) { + throw new Error(`Failed to remove dependency: ${removeDepResult.stderr}`); + } + logger.success('✓ Removed dependency successfully'); + + // Test 13: Validate dependencies + logger.info('\nTest 13: Validate dependencies'); + const validateResult = await helpers.taskMaster('validate-dependencies', [], { cwd: testDir }); + if (validateResult.exitCode !== 0) { + throw new Error(`Dependency validation failed: ${validateResult.stderr}`); + } + logger.success('✓ Dependency validation successful'); + + // Test 14: Update task description + logger.info('\nTest 14: Update task description'); + const updateResult = await helpers.taskMaster('update-task', [manualTaskId, '--description', 'Write comprehensive unit tests'], { cwd: testDir }); + if (updateResult.exitCode !== 0) { + throw new Error(`Failed to update task: ${updateResult.stderr}`); + } + logger.success('✓ Updated task description successfully'); + + // Test 15: Add subtask + logger.info('\nTest 15: Add subtask'); + const subtaskResult = await helpers.taskMaster('add-subtask', [manualTaskId, 'Write test for login'], { cwd: testDir }); + if (subtaskResult.exitCode !== 0) { + throw new Error(`Failed to add subtask: ${subtaskResult.stderr}`); + } + const subtaskId = helpers.extractTaskId(subtaskResult.stdout) || '1.1'; + logger.success(`✓ Added subtask with ID: ${subtaskId}`); + + // Test 16: Verify subtask relationship + logger.info('\nTest 16: Verify subtask relationship'); + const showResult3 = await helpers.taskMaster('show', [manualTaskId], { cwd: testDir }); + if (showResult3.exitCode !== 0) { + throw new Error(`Show task failed: ${showResult3.stderr}`); + } + if (!showResult3.stdout.includes('Subtasks:')) { + throw new Error('Subtasks section not shown in parent task'); + } + logger.success('✓ Subtask relationship verified'); + + // Test 17: Set task status to in_progress + logger.info('\nTest 17: Set task status to in_progress'); + const statusResult1 = await helpers.taskMaster('set-status', [manualTaskId, 'in_progress'], { cwd: testDir }); + if (statusResult1.exitCode !== 0) { + throw new Error(`Failed to update task status: ${statusResult1.stderr}`); + } + logger.success('✓ Set task status to in_progress'); + + // Test 18: Set task status to completed + logger.info('\nTest 18: Set task status to completed'); + const statusResult2 = await helpers.taskMaster('set-status', [dbTaskId, 'completed'], { cwd: testDir }); + if (statusResult2.exitCode !== 0) { + throw new Error(`Failed to complete task: ${statusResult2.stderr}`); + } + logger.success('✓ Set task status to completed'); + + // Test 19: List tasks with status filter + logger.info('\nTest 19: List tasks by status'); + const listStatusResult = await helpers.taskMaster('list', ['--status', 'completed'], { cwd: testDir }); + if (listStatusResult.exitCode !== 0) { + throw new Error(`List by status failed: ${listStatusResult.stderr}`); + } + if (!listStatusResult.stdout.includes('Create database schema')) { + throw new Error('Completed task not shown in filtered list'); + } + logger.success('✓ List tasks by status successful'); + + // Test 20: Remove single task + logger.info('\nTest 20: Remove single task'); + const removeResult1 = await helpers.taskMaster('remove-task', [dbTaskId], { cwd: testDir }); + if (removeResult1.exitCode !== 0) { + throw new Error(`Failed to remove task: ${removeResult1.stderr}`); + } + logger.success('✓ Removed single task successfully'); + + // Test 21: Remove multiple tasks + logger.info('\nTest 21: Remove multiple tasks'); + const removeResult2 = await helpers.taskMaster('remove-task', [manualTaskId, aiTaskId], { cwd: testDir }); + if (removeResult2.exitCode !== 0) { + throw new Error(`Failed to remove multiple tasks: ${removeResult2.stderr}`); + } + logger.success('✓ Removed multiple tasks successfully'); + + // Test 22: Verify tasks were removed + logger.info('\nTest 22: Verify tasks were removed'); + const listResult3 = await helpers.taskMaster('list', [], { cwd: testDir }); + if (listResult3.exitCode !== 0) { + throw new Error(`List command failed: ${listResult3.stderr}`); + } + // Check that our specific task IDs are no longer in the list + const stillHasTask11 = new RegExp(`\\b${manualTaskId}\\b`).test(listResult3.stdout); + const stillHasTask12 = new RegExp(`\\b${aiTaskId}\\b`).test(listResult3.stdout); + const stillHasTask13 = new RegExp(`\\b${dbTaskId}\\b`).test(listResult3.stdout); + + if (stillHasTask11 || stillHasTask12 || stillHasTask13) { + throw new Error('Removed task IDs still appear in list'); + } + logger.success('✓ Verified tasks were removed'); + + // Test 23: Fix dependencies (cleanup) + logger.info('\nTest 23: Fix dependencies'); + const fixDepsResult = await helpers.taskMaster('fix-dependencies', [], { cwd: testDir }); + if (fixDepsResult.exitCode !== 0) { + // Non-critical, just log + logger.warning(`Fix dependencies had issues: ${fixDepsResult.stderr}`); + } else { + logger.success('✓ Fix dependencies command executed'); + } + + logger.info('\n✅ All core task operations tests passed!'); + + } catch (error) { + results.status = 'failed'; + results.errors.push({ + test: 'core operations', + error: error.message, + stack: error.stack + }); + logger.error(`Core operations test failed: ${error.message}`); + } + + return results; +} \ No newline at end of file diff --git a/tests/e2e/tests/providers.test.js b/tests/e2e/tests/providers.test.js new file mode 100644 index 00000000..6ac22888 --- /dev/null +++ b/tests/e2e/tests/providers.test.js @@ -0,0 +1,164 @@ +/** + * Multi-provider functionality test module + * Tests add-task operation across all configured providers + */ + +export default async function testProviders(logger, helpers, context) { + const { testDir, config } = context; + const results = { + status: 'passed', + errors: [], + providerComparison: {}, + summary: { + totalProviders: 0, + successfulProviders: 0, + failedProviders: 0, + averageExecutionTime: 0, + successRate: '0%' + } + }; + + try { + logger.info('Starting multi-provider tests...'); + + const providers = config.providers; + const standardPrompt = config.prompts.addTask; + + results.summary.totalProviders = providers.length; + let totalExecutionTime = 0; + + // Process providers in batches to avoid rate limits + const batchSize = 3; + for (let i = 0; i < providers.length; i += batchSize) { + const batch = providers.slice(i, i + batchSize); + + const batchPromises = batch.map(async (provider) => { + const providerResult = { + status: 'failed', + taskId: null, + executionTime: 0, + subtaskCount: 0, + features: { + hasTitle: false, + hasDescription: false, + hasSubtasks: false, + hasDependencies: false + }, + error: null, + taskDetails: null + }; + + const startTime = Date.now(); + + try { + logger.info(`\nTesting provider: ${provider.name} with model: ${provider.model}`); + + // Step 1: Set the main model for this provider + logger.info(`Setting model to ${provider.model}...`); + const setModelResult = await helpers.taskMaster('models', ['--set-main', provider.model], { cwd: testDir }); + if (setModelResult.exitCode !== 0) { + throw new Error(`Failed to set model for ${provider.name}: ${setModelResult.stderr}`); + } + + // Step 2: Execute add-task with standard prompt + logger.info(`Adding task with ${provider.name}...`); + const addTaskArgs = ['--prompt', standardPrompt]; + if (provider.flags && provider.flags.length > 0) { + addTaskArgs.push(...provider.flags); + } + + const addTaskResult = await helpers.taskMaster('add-task', addTaskArgs, { + cwd: testDir, + timeout: 120000 // 2 minutes timeout for AI tasks + }); + + if (addTaskResult.exitCode !== 0) { + throw new Error(`Add-task failed: ${addTaskResult.stderr}`); + } + + // Step 3: Extract task ID from output + const taskId = helpers.extractTaskId(addTaskResult.stdout); + if (!taskId) { + throw new Error(`Failed to extract task ID from output`); + } + providerResult.taskId = taskId; + logger.success(`✓ Created task ${taskId} with ${provider.name}`); + + // Step 4: Get task details + const showResult = await helpers.taskMaster('show', [taskId], { cwd: testDir }); + if (showResult.exitCode === 0) { + providerResult.taskDetails = showResult.stdout; + + // Analyze task features + providerResult.features.hasTitle = showResult.stdout.includes('Title:') || + showResult.stdout.includes('Task:'); + providerResult.features.hasDescription = showResult.stdout.includes('Description:'); + providerResult.features.hasSubtasks = showResult.stdout.includes('Subtasks:'); + providerResult.features.hasDependencies = showResult.stdout.includes('Dependencies:'); + + // Count subtasks + const subtaskMatches = showResult.stdout.match(/\d+\.\d+/g); + providerResult.subtaskCount = subtaskMatches ? subtaskMatches.length : 0; + } + + providerResult.status = 'success'; + results.summary.successfulProviders++; + + } catch (error) { + providerResult.status = 'failed'; + providerResult.error = error.message; + results.summary.failedProviders++; + logger.error(`${provider.name} test failed: ${error.message}`); + } + + providerResult.executionTime = Date.now() - startTime; + totalExecutionTime += providerResult.executionTime; + + results.providerComparison[provider.name] = providerResult; + }); + + // Wait for batch to complete + await Promise.all(batchPromises); + + // Small delay between batches to avoid rate limits + if (i + batchSize < providers.length) { + logger.info('Waiting 2 seconds before next batch...'); + await helpers.wait(2000); + } + } + + // Calculate summary statistics + results.summary.averageExecutionTime = Math.round(totalExecutionTime / providers.length); + results.summary.successRate = `${Math.round((results.summary.successfulProviders / results.summary.totalProviders) * 100)}%`; + + // Log summary + logger.info('\n=== Provider Test Summary ==='); + logger.info(`Total providers tested: ${results.summary.totalProviders}`); + logger.info(`Successful: ${results.summary.successfulProviders}`); + logger.info(`Failed: ${results.summary.failedProviders}`); + logger.info(`Success rate: ${results.summary.successRate}`); + logger.info(`Average execution time: ${results.summary.averageExecutionTime}ms`); + + // Determine overall status + if (results.summary.failedProviders === 0) { + logger.success('✅ All provider tests passed!'); + } else if (results.summary.successfulProviders > 0) { + results.status = 'partial'; + logger.warning(`⚠️ ${results.summary.failedProviders} provider(s) failed`); + } else { + results.status = 'failed'; + logger.error('❌ All provider tests failed'); + } + + } catch (error) { + results.status = 'failed'; + results.errors.push({ + test: 'provider tests', + error: error.message, + stack: error.stack + }); + logger.error(`Provider tests failed: ${error.message}`); + } + + return results; +} \ No newline at end of file diff --git a/tests/e2e/tests/setup.test.js b/tests/e2e/tests/setup.test.js new file mode 100644 index 00000000..f3ffa030 --- /dev/null +++ b/tests/e2e/tests/setup.test.js @@ -0,0 +1,266 @@ +import { mkdirSync, existsSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { testConfig } from '../config/test-config.js'; + +/** + * Setup test module that handles initialization, PRD parsing, and complexity analysis + * @param {Object} logger - TestLogger instance + * @param {Object} helpers - TestHelpers instance + * @returns {Promise} Test results object with status and directory path + */ +export async function runSetupTest(logger, helpers) { + const testResults = { + status: 'pending', + testDir: null, + steps: { + createDirectory: false, + linkGlobally: false, + copyEnv: false, + initialize: false, + parsePrd: false, + analyzeComplexity: false, + generateReport: false + }, + errors: [], + prdPath: null, + complexityReport: null + }; + + try { + // Step 1: Create test directory with timestamp + logger.step('Creating test directory'); + const timestamp = new Date().toISOString().replace(/[:.]/g, '').replace('T', '_').slice(0, -1); + const testDir = join(testConfig.paths.baseTestDir, `run_${timestamp}`); + + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + + testResults.testDir = testDir; + testResults.steps.createDirectory = true; + logger.success(`Test directory created: ${testDir}`); + + // Step 2: Link task-master globally + logger.step('Linking task-master globally'); + const linkResult = await helpers.executeCommand('npm', ['link'], { + cwd: testConfig.paths.projectRoot, + timeout: 60000 + }); + + if (linkResult.exitCode === 0) { + testResults.steps.linkGlobally = true; + logger.success('Task-master linked globally'); + } else { + throw new Error(`Failed to link task-master: ${linkResult.stderr}`); + } + + // Step 3: Copy .env file + logger.step('Copying .env file to test directory'); + const envSourcePath = testConfig.paths.mainEnvFile; + const envDestPath = join(testDir, '.env'); + + if (helpers.fileExists(envSourcePath)) { + if (helpers.copyFile(envSourcePath, envDestPath)) { + testResults.steps.copyEnv = true; + logger.success('.env file copied successfully'); + } else { + throw new Error('Failed to copy .env file'); + } + } else { + logger.warning('.env file not found at source, proceeding without it'); + } + + // Step 4: Initialize project with task-master init + logger.step('Initializing project with task-master'); + const initResult = await helpers.taskMaster('init', [ + '-y', + '--name="E2E Test ' + testDir.split('/').pop() + '"', + '--description="Automated E2E test run"' + ], { + cwd: testDir, + timeout: 120000 + }); + + if (initResult.exitCode === 0) { + testResults.steps.initialize = true; + logger.success('Project initialized successfully'); + + // Save init debug log if available + const initDebugPath = join(testDir, 'init-debug.log'); + if (existsSync(initDebugPath)) { + logger.info('Init debug log saved'); + } + } else { + throw new Error(`Initialization failed: ${initResult.stderr}`); + } + + // Step 5: Parse PRD from sample file + logger.step('Parsing PRD from sample file'); + + // First, copy the sample PRD to the test directory + const prdSourcePath = testConfig.paths.samplePrdSource; + const prdDestPath = join(testDir, 'prd.txt'); + testResults.prdPath = prdDestPath; + + if (!helpers.fileExists(prdSourcePath)) { + // If sample PRD doesn't exist in fixtures, use the example PRD + const examplePrdPath = join(testConfig.paths.projectRoot, 'assets/example_prd.txt'); + if (helpers.fileExists(examplePrdPath)) { + helpers.copyFile(examplePrdPath, prdDestPath); + logger.info('Using example PRD file'); + } else { + // Create a minimal PRD for testing + const minimalPrd = ` +# Overview +A simple task management system for developers. + +# Core Features +- Task creation and management +- Task dependencies +- Status tracking +- Task prioritization + +# Technical Architecture +- Node.js backend +- REST API +- JSON data storage +- CLI interface + +# Development Roadmap +Phase 1: Core functionality +- Initialize project structure +- Implement task CRUD operations +- Add dependency management + +Phase 2: Enhanced features +- Add task prioritization +- Implement search functionality +- Add export capabilities + +# Logical Dependency Chain +1. Project setup and initialization +2. Core data models +3. Basic CRUD operations +4. Dependency system +5. CLI interface +6. Advanced features +`; + + writeFileSync(prdDestPath, minimalPrd); + logger.info('Created minimal PRD for testing'); + } + } else { + helpers.copyFile(prdSourcePath, prdDestPath); + } + + // Parse the PRD + const parsePrdResult = await helpers.taskMaster('parse-prd', ['prd.txt'], { + cwd: testDir, + timeout: 180000 + }); + + if (parsePrdResult.exitCode === 0) { + testResults.steps.parsePrd = true; + logger.success('PRD parsed successfully'); + + // Extract task count from output + const taskCountMatch = parsePrdResult.stdout.match(/(\d+) tasks? created/i); + if (taskCountMatch) { + logger.info(`Created ${taskCountMatch[1]} tasks from PRD`); + } + } else { + throw new Error(`PRD parsing failed: ${parsePrdResult.stderr}`); + } + + // Step 6: Run complexity analysis + logger.step('Running complexity analysis on parsed tasks'); + // Ensure reports directory exists + const reportsDir = join(testDir, '.taskmaster/reports'); + if (!existsSync(reportsDir)) { + mkdirSync(reportsDir, { recursive: true }); + } + const analyzeResult = await helpers.taskMaster('analyze-complexity', ['--research', '--output', '.taskmaster/reports/task-complexity-report.json'], { + cwd: testDir, + timeout: 240000 + }); + + if (analyzeResult.exitCode === 0) { + testResults.steps.analyzeComplexity = true; + logger.success('Complexity analysis completed'); + + // Extract complexity information from output + const complexityMatch = analyzeResult.stdout.match(/Total Complexity Score: ([\d.]+)/); + if (complexityMatch) { + logger.info(`Total complexity score: ${complexityMatch[1]}`); + } + } else { + throw new Error(`Complexity analysis failed: ${analyzeResult.stderr}`); + } + + // Step 7: Generate complexity report + logger.step('Generating complexity report'); + const reportResult = await helpers.taskMaster('complexity-report', [], { + cwd: testDir, + timeout: 60000 + }); + + if (reportResult.exitCode === 0) { + testResults.steps.generateReport = true; + logger.success('Complexity report generated'); + + // Check if complexity report file was created (not needed since complexity-report reads from the standard location) + const reportPath = join(testDir, '.taskmaster/reports/task-complexity-report.json'); + if (helpers.fileExists(reportPath)) { + testResults.complexityReport = helpers.readJson(reportPath); + logger.info('Complexity report saved to task-complexity-report.json'); + + // Log summary if available + if (testResults.complexityReport && testResults.complexityReport.summary) { + const summary = testResults.complexityReport.summary; + logger.info(`Tasks analyzed: ${summary.totalTasks || 0}`); + logger.info(`Average complexity: ${summary.averageComplexity || 0}`); + } + } + } else { + logger.warning(`Complexity report generation had issues: ${reportResult.stderr}`); + // Don't fail the test for report generation issues + testResults.steps.generateReport = true; + } + + // Verify tasks.json was created + const tasksJsonPath = join(testDir, '.taskmaster/tasks/tasks.json'); + if (helpers.fileExists(tasksJsonPath)) { + const taskCount = helpers.getTaskCount(tasksJsonPath); + logger.info(`Verified tasks.json exists with ${taskCount} tasks`); + } else { + throw new Error('tasks.json was not created'); + } + + // All steps completed successfully + testResults.status = 'success'; + logger.success('Setup test completed successfully'); + + } catch (error) { + testResults.status = 'failed'; + testResults.errors.push(error.message); + logger.error(`Setup test failed: ${error.message}`); + + // Log which steps completed + logger.info('Completed steps:'); + Object.entries(testResults.steps).forEach(([step, completed]) => { + if (completed) { + logger.info(` ✓ ${step}`); + } else { + logger.info(` ✗ ${step}`); + } + }); + } + + // Flush logs before returning + logger.flush(); + + return testResults; +} + +// Export default for direct execution +export default runSetupTest; \ No newline at end of file diff --git a/tests/e2e/utils/error-handler.js b/tests/e2e/utils/error-handler.js new file mode 100644 index 00000000..4a4c6343 --- /dev/null +++ b/tests/e2e/utils/error-handler.js @@ -0,0 +1,236 @@ +import { writeFileSync } from 'fs'; +import { join } from 'path'; +import chalk from 'chalk'; + +export class ErrorHandler { + constructor(logger) { + this.logger = logger; + this.errors = []; + this.warnings = []; + } + + /** + * Handle and categorize errors + */ + handleError(error, context = {}) { + const errorInfo = { + timestamp: new Date().toISOString(), + message: error.message || 'Unknown error', + stack: error.stack, + context, + type: this.categorizeError(error) + }; + + this.errors.push(errorInfo); + this.logger.error(`[${errorInfo.type}] ${errorInfo.message}`); + + if (context.critical) { + throw error; + } + + return errorInfo; + } + + /** + * Add a warning + */ + addWarning(message, context = {}) { + const warning = { + timestamp: new Date().toISOString(), + message, + context + }; + + this.warnings.push(warning); + this.logger.warning(message); + } + + /** + * Categorize error types + */ + categorizeError(error) { + const message = error.message.toLowerCase(); + + if (message.includes('command not found') || message.includes('not found')) { + return 'DEPENDENCY_ERROR'; + } + if (message.includes('permission') || message.includes('access denied')) { + return 'PERMISSION_ERROR'; + } + if (message.includes('timeout')) { + return 'TIMEOUT_ERROR'; + } + if (message.includes('api') || message.includes('rate limit')) { + return 'API_ERROR'; + } + if (message.includes('json') || message.includes('parse')) { + return 'PARSE_ERROR'; + } + if (message.includes('file') || message.includes('directory')) { + return 'FILE_ERROR'; + } + + return 'GENERAL_ERROR'; + } + + /** + * Get error summary + */ + getSummary() { + const errorsByType = {}; + + this.errors.forEach(error => { + if (!errorsByType[error.type]) { + errorsByType[error.type] = []; + } + errorsByType[error.type].push(error); + }); + + return { + totalErrors: this.errors.length, + totalWarnings: this.warnings.length, + errorsByType, + criticalErrors: this.errors.filter(e => e.context.critical), + recentErrors: this.errors.slice(-5) + }; + } + + /** + * Generate error report + */ + generateReport(outputPath) { + const summary = this.getSummary(); + const report = { + generatedAt: new Date().toISOString(), + summary: { + totalErrors: summary.totalErrors, + totalWarnings: summary.totalWarnings, + errorTypes: Object.keys(summary.errorsByType) + }, + errors: this.errors, + warnings: this.warnings, + recommendations: this.generateRecommendations(summary) + }; + + writeFileSync(outputPath, JSON.stringify(report, null, 2)); + return report; + } + + /** + * Generate recommendations based on errors + */ + generateRecommendations(summary) { + const recommendations = []; + + if (summary.errorsByType.DEPENDENCY_ERROR) { + recommendations.push({ + type: 'DEPENDENCY', + message: 'Install missing dependencies using npm install or check PATH', + errors: summary.errorsByType.DEPENDENCY_ERROR.length + }); + } + + if (summary.errorsByType.PERMISSION_ERROR) { + recommendations.push({ + type: 'PERMISSION', + message: 'Check file permissions or run with appropriate privileges', + errors: summary.errorsByType.PERMISSION_ERROR.length + }); + } + + if (summary.errorsByType.API_ERROR) { + recommendations.push({ + type: 'API', + message: 'Check API keys, rate limits, or network connectivity', + errors: summary.errorsByType.API_ERROR.length + }); + } + + if (summary.errorsByType.TIMEOUT_ERROR) { + recommendations.push({ + type: 'TIMEOUT', + message: 'Consider increasing timeout values or optimizing slow operations', + errors: summary.errorsByType.TIMEOUT_ERROR.length + }); + } + + return recommendations; + } + + /** + * Display error summary in console + */ + displaySummary() { + const summary = this.getSummary(); + + if (summary.totalErrors === 0 && summary.totalWarnings === 0) { + console.log(chalk.green('✅ No errors or warnings detected')); + return; + } + + console.log(chalk.red.bold(`\n🚨 Error Summary:`)); + console.log(chalk.red(` Total Errors: ${summary.totalErrors}`)); + console.log(chalk.yellow(` Total Warnings: ${summary.totalWarnings}`)); + + if (summary.totalErrors > 0) { + console.log(chalk.red.bold('\n Error Types:')); + Object.entries(summary.errorsByType).forEach(([type, errors]) => { + console.log(chalk.red(` - ${type}: ${errors.length}`)); + }); + + if (summary.criticalErrors.length > 0) { + console.log(chalk.red.bold(`\n ⚠️ Critical Errors: ${summary.criticalErrors.length}`)); + summary.criticalErrors.forEach(error => { + console.log(chalk.red(` - ${error.message}`)); + }); + } + } + + const recommendations = this.generateRecommendations(summary); + if (recommendations.length > 0) { + console.log(chalk.yellow.bold('\n💡 Recommendations:')); + recommendations.forEach(rec => { + console.log(chalk.yellow(` - ${rec.message}`)); + }); + } + } + + /** + * Clear all errors and warnings + */ + clear() { + this.errors = []; + this.warnings = []; + } +} + +/** + * Global error handler for uncaught exceptions + */ +export function setupGlobalErrorHandlers(errorHandler, logger) { + process.on('uncaughtException', (error) => { + logger.error(`Uncaught Exception: ${error.message}`); + errorHandler.handleError(error, { critical: true, source: 'uncaughtException' }); + process.exit(1); + }); + + process.on('unhandledRejection', (reason, promise) => { + logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`); + errorHandler.handleError(new Error(String(reason)), { + critical: false, + source: 'unhandledRejection' + }); + }); + + process.on('SIGINT', () => { + logger.info('\nReceived SIGINT, shutting down gracefully...'); + errorHandler.displaySummary(); + process.exit(130); + }); + + process.on('SIGTERM', () => { + logger.info('\nReceived SIGTERM, shutting down...'); + errorHandler.displaySummary(); + process.exit(143); + }); +} \ No newline at end of file diff --git a/tests/e2e/utils/llm-analyzer.js b/tests/e2e/utils/llm-analyzer.js new file mode 100644 index 00000000..fa345e5d --- /dev/null +++ b/tests/e2e/utils/llm-analyzer.js @@ -0,0 +1,168 @@ +import { readFileSync } from 'fs'; +import fetch from 'node-fetch'; + +export class LLMAnalyzer { + constructor(config, logger) { + this.config = config; + this.logger = logger; + this.apiKey = process.env.ANTHROPIC_API_KEY; + this.apiEndpoint = 'https://api.anthropic.com/v1/messages'; + } + + async analyzeLog(logFile, providerSummaryFile = null) { + if (!this.config.llmAnalysis.enabled) { + this.logger.info('LLM analysis is disabled in configuration'); + return null; + } + + if (!this.apiKey) { + this.logger.error('ANTHROPIC_API_KEY not found in environment'); + return null; + } + + try { + const logContent = readFileSync(logFile, 'utf8'); + const prompt = this.buildAnalysisPrompt(logContent, providerSummaryFile); + + const response = await this.callLLM(prompt); + const analysis = this.parseResponse(response); + + // Calculate and log cost + if (response.usage) { + const cost = this.calculateCost(response.usage); + this.logger.addCost(cost); + this.logger.info(`LLM Analysis AI Cost: $${cost.toFixed(6)} USD`); + } + + return analysis; + } catch (error) { + this.logger.error(`LLM analysis failed: ${error.message}`); + return null; + } + } + + buildAnalysisPrompt(logContent, providerSummaryFile) { + let providerSummary = ''; + if (providerSummaryFile) { + try { + providerSummary = readFileSync(providerSummaryFile, 'utf8'); + } catch (error) { + this.logger.warning(`Could not read provider summary file: ${error.message}`); + } + } + + return `Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially. + +Your goal is to: +1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files). +2. **Specifically analyze the Multi-Provider Add-Task Test Sequence:** + a. Identify which providers were tested for \`add-task\`. Look for log steps like "Testing Add-Task with Provider: ..." and the summary log 'provider_add_task_summary.log'. + b. For each tested provider, determine if \`add-task\` succeeded or failed. Note the created task ID if successful. + c. Review the corresponding \`add_task_show_output__id_.log\` file (if created) for each successful \`add-task\` execution. + d. **Compare the quality and completeness** of the task generated by each successful provider based on their \`show\` output. Assign a score (e.g., 1-10, 10 being best) based on relevance to the prompt, detail level, and correctness. + e. Note any providers where \`add-task\` failed or where the task ID could not be extracted. +3. Identify any general explicit "[ERROR]" messages or stack traces throughout the *entire* log. +4. Identify any potential warnings or unusual output that might indicate a problem even if not marked as an explicit error. +5. Provide an overall assessment of the test run's health based *only* on the log content. + +${providerSummary ? `\nProvider Summary:\n${providerSummary}\n` : ''} + +Return your analysis **strictly** in the following JSON format. Do not include any text outside of the JSON structure: + +{ + "overall_status": "Success|Failure|Warning", + "verified_steps": [ "Initialization", "PRD Parsing", /* ...other general steps observed... */ ], + "provider_add_task_comparison": { + "prompt_used": "... (extract from log if possible or state 'standard auth prompt') ...", + "provider_results": { + "anthropic": { "status": "Success|Failure|ID_Extraction_Failed|Set_Model_Failed", "task_id": "...", "score": "X/10 | N/A", "notes": "..." }, + "openai": { "status": "Success|Failure|...", "task_id": "...", "score": "X/10 | N/A", "notes": "..." }, + /* ... include all tested providers ... */ + }, + "comparison_summary": "Brief overall comparison of generated tasks..." + }, + "detected_issues": [ { "severity": "Error|Warning|Anomaly", "description": "...", "log_context": "[Optional, short snippet from log near the issue]" } ], + "llm_summary_points": [ "Overall summary point 1", "Provider comparison highlight", "Any major issues noted" ] +} + +Here is the main log content: + +${logContent}`; + } + + async callLLM(prompt) { + const payload = { + model: this.config.llmAnalysis.model, + max_tokens: this.config.llmAnalysis.maxTokens, + messages: [ + { role: 'user', content: prompt } + ] + }; + + const response = await fetch(this.apiEndpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': this.apiKey, + 'anthropic-version': '2023-06-01' + }, + body: JSON.stringify(payload) + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`LLM API call failed: ${response.status} - ${error}`); + } + + return response.json(); + } + + parseResponse(response) { + try { + const content = response.content[0].text; + const jsonStart = content.indexOf('{'); + const jsonEnd = content.lastIndexOf('}'); + + if (jsonStart === -1 || jsonEnd === -1) { + throw new Error('No JSON found in response'); + } + + const jsonString = content.substring(jsonStart, jsonEnd + 1); + return JSON.parse(jsonString); + } catch (error) { + this.logger.error(`Failed to parse LLM response: ${error.message}`); + return null; + } + } + + calculateCost(usage) { + const modelCosts = { + 'claude-3-7-sonnet-20250219': { + input: 3.00, // per 1M tokens + output: 15.00 // per 1M tokens + } + }; + + const costs = modelCosts[this.config.llmAnalysis.model] || { input: 0, output: 0 }; + const inputCost = (usage.input_tokens / 1000000) * costs.input; + const outputCost = (usage.output_tokens / 1000000) * costs.output; + + return inputCost + outputCost; + } + + formatReport(analysis) { + if (!analysis) return null; + + const report = { + title: 'TASKMASTER E2E Test Analysis Report', + timestamp: new Date().toISOString(), + status: analysis.overall_status, + summary: analysis.llm_summary_points, + verifiedSteps: analysis.verified_steps, + providerComparison: analysis.provider_add_task_comparison, + issues: analysis.detected_issues + }; + + return report; + } +} \ No newline at end of file diff --git a/tests/e2e/utils/logger.js b/tests/e2e/utils/logger.js new file mode 100644 index 00000000..ded9f1fa --- /dev/null +++ b/tests/e2e/utils/logger.js @@ -0,0 +1,124 @@ +import { writeFileSync, mkdirSync, existsSync } from 'fs'; +import { join } from 'path'; +import chalk from 'chalk'; + +export class TestLogger { + constructor(logDir, testRunId) { + this.logDir = logDir; + this.testRunId = testRunId; + this.startTime = Date.now(); + this.stepCount = 0; + this.logFile = join(logDir, `e2e_run_${testRunId}.log`); + this.logBuffer = []; + this.totalCost = 0; + + // Ensure log directory exists + if (!existsSync(logDir)) { + mkdirSync(logDir, { recursive: true }); + } + } + + formatDuration(milliseconds) { + const totalSeconds = Math.floor(milliseconds / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}m${seconds.toString().padStart(2, '0')}s`; + } + + getElapsedTime() { + return this.formatDuration(Date.now() - this.startTime); + } + + formatLogEntry(level, message) { + const timestamp = new Date().toISOString(); + const elapsed = this.getElapsedTime(); + return `[${level}] [${elapsed}] ${timestamp} ${message}`; + } + + log(level, message, options = {}) { + const formattedMessage = this.formatLogEntry(level, message); + + // Add to buffer + this.logBuffer.push(formattedMessage); + + // Console output with colors + let coloredMessage = formattedMessage; + switch (level) { + case 'INFO': + coloredMessage = chalk.blue(formattedMessage); + break; + case 'SUCCESS': + coloredMessage = chalk.green(formattedMessage); + break; + case 'ERROR': + coloredMessage = chalk.red(formattedMessage); + break; + case 'WARNING': + coloredMessage = chalk.yellow(formattedMessage); + break; + } + + console.log(coloredMessage); + + // Write to file if immediate flush requested + if (options.flush) { + this.flush(); + } + } + + info(message) { + this.log('INFO', message); + } + + success(message) { + this.log('SUCCESS', message); + } + + error(message) { + this.log('ERROR', message); + } + + warning(message) { + this.log('WARNING', message); + } + + step(message) { + this.stepCount++; + const separator = '='.repeat(45); + this.log('STEP', `\n${separator}\n STEP ${this.stepCount}: ${message}\n${separator}`); + } + + addCost(cost) { + if (typeof cost === 'number' && !isNaN(cost)) { + this.totalCost += cost; + } + } + + extractAndAddCost(output) { + const costRegex = /Est\. Cost: \$(\d+\.\d+)/g; + let match; + while ((match = costRegex.exec(output)) !== null) { + const cost = parseFloat(match[1]); + this.addCost(cost); + } + } + + flush() { + writeFileSync(this.logFile, this.logBuffer.join('\n'), 'utf8'); + } + + getSummary() { + const duration = this.formatDuration(Date.now() - this.startTime); + const successCount = this.logBuffer.filter(line => line.includes('[SUCCESS]')).length; + const errorCount = this.logBuffer.filter(line => line.includes('[ERROR]')).length; + + return { + duration, + totalSteps: this.stepCount, + successCount, + errorCount, + totalCost: this.totalCost.toFixed(6), + logFile: this.logFile + }; + } +} \ No newline at end of file diff --git a/tests/e2e/utils/test-helpers.js b/tests/e2e/utils/test-helpers.js new file mode 100644 index 00000000..db4f0a67 --- /dev/null +++ b/tests/e2e/utils/test-helpers.js @@ -0,0 +1,187 @@ +import { spawn } from 'child_process'; +import { readFileSync, existsSync, copyFileSync } from 'fs'; +import { join } from 'path'; + +export class TestHelpers { + constructor(logger) { + this.logger = logger; + } + + /** + * Execute a command and return output + * @param {string} command - Command to execute + * @param {string[]} args - Command arguments + * @param {Object} options - Execution options + * @returns {Promise<{stdout: string, stderr: string, exitCode: number}>} + */ + async executeCommand(command, args = [], options = {}) { + return new Promise((resolve) => { + const spawnOptions = { + cwd: options.cwd || process.cwd(), + env: { ...process.env, ...options.env }, + shell: true + }; + + // When using shell: true, pass the full command as a single string + const fullCommand = args.length > 0 ? `${command} ${args.join(' ')}` : command; + const child = spawn(fullCommand, [], spawnOptions); + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (exitCode) => { + const output = stdout + stderr; + + // Extract and log costs + this.logger.extractAndAddCost(output); + + resolve({ stdout, stderr, exitCode }); + }); + + // Handle timeout + if (options.timeout) { + setTimeout(() => { + child.kill('SIGTERM'); + }, options.timeout); + } + }); + } + + /** + * Execute task-master command + * @param {string} subcommand - Task-master subcommand + * @param {string[]} args - Command arguments + * @param {Object} options - Execution options + */ + async taskMaster(subcommand, args = [], options = {}) { + const fullArgs = [subcommand, ...args]; + this.logger.info(`Executing: task-master ${fullArgs.join(' ')}`); + + const result = await this.executeCommand('task-master', fullArgs, options); + + if (result.exitCode !== 0 && !options.allowFailure) { + this.logger.error(`Command failed with exit code ${result.exitCode}`); + this.logger.error(`stderr: ${result.stderr}`); + } + + return result; + } + + /** + * Check if a file exists + */ + fileExists(filePath) { + return existsSync(filePath); + } + + /** + * Read JSON file + */ + readJson(filePath) { + try { + const content = readFileSync(filePath, 'utf8'); + return JSON.parse(content); + } catch (error) { + this.logger.error(`Failed to read JSON file ${filePath}: ${error.message}`); + return null; + } + } + + /** + * Copy file + */ + copyFile(source, destination) { + try { + copyFileSync(source, destination); + return true; + } catch (error) { + this.logger.error(`Failed to copy file from ${source} to ${destination}: ${error.message}`); + return false; + } + } + + /** + * Wait for a specified duration + */ + async wait(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); + } + + /** + * Verify task exists in tasks.json + */ + verifyTaskExists(tasksFile, taskId, tagName = 'master') { + const tasks = this.readJson(tasksFile); + if (!tasks || !tasks[tagName]) return false; + + return tasks[tagName].tasks.some(task => task.id === taskId); + } + + /** + * Get task count for a tag + */ + getTaskCount(tasksFile, tagName = 'master') { + const tasks = this.readJson(tasksFile); + if (!tasks || !tasks[tagName]) return 0; + + return tasks[tagName].tasks.length; + } + + /** + * Extract task ID from command output + */ + extractTaskId(output) { + const patterns = [ + /✓ Added new task #(\d+(?:\.\d+)?)/, + /✅ New task created successfully:.*?(\d+(?:\.\d+)?)/, + /Task (\d+(?:\.\d+)?) Created Successfully/ + ]; + + for (const pattern of patterns) { + const match = output.match(pattern); + if (match) { + return match[1]; + } + } + + return null; + } + + /** + * Run multiple async operations in parallel + */ + async runParallel(operations) { + return Promise.all(operations); + } + + /** + * Run operations with concurrency limit + */ + async runWithConcurrency(operations, limit = 3) { + const results = []; + const executing = []; + + for (const operation of operations) { + const promise = operation().then(result => { + executing.splice(executing.indexOf(promise), 1); + return result; + }); + + results.push(promise); + executing.push(promise); + + if (executing.length >= limit) { + await Promise.race(executing); + } + } + + return Promise.all(results); + } +} \ No newline at end of file