feat: Introduce modern E2E test suite for Task Master AI
This commit is contained in:
Ralph Khreish
392
tests/e2e/run-e2e-tests.js
Executable file
392
tests/e2e/run-e2e-tests.js
Executable file
@@ -0,0 +1,392 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import process from 'process';
|
||||
import chalk from 'chalk';
|
||||
import boxen from 'boxen';
|
||||
import { TestLogger } from './utils/logger.js';
|
||||
import { TestHelpers } from './utils/test-helpers.js';
|
||||
import { LLMAnalyzer } from './utils/llm-analyzer.js';
|
||||
import { testConfig, testGroups } from './config/test-config.js';
|
||||
import {
|
||||
ParallelTestRunner,
|
||||
SequentialTestRunner
|
||||
} from './runners/parallel-runner.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Parse command line arguments
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
const options = {
|
||||
skipVerification: false,
|
||||
analyzeLog: false,
|
||||
logFile: null,
|
||||
parallel: true,
|
||||
groups: null,
|
||||
testDir: null
|
||||
};
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
switch (args[i]) {
|
||||
case '--skip-verification':
|
||||
options.skipVerification = true;
|
||||
break;
|
||||
case '--analyze-log':
|
||||
options.analyzeLog = true;
|
||||
if (args[i + 1] && !args[i + 1].startsWith('--')) {
|
||||
options.logFile = args[i + 1];
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
case '--sequential':
|
||||
options.parallel = false;
|
||||
break;
|
||||
case '--groups':
|
||||
if (args[i + 1] && !args[i + 1].startsWith('--')) {
|
||||
options.groups = args[i + 1].split(',');
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
case '--test-dir':
|
||||
if (args[i + 1] && !args[i + 1].startsWith('--')) {
|
||||
options.testDir = args[i + 1];
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
case '--help':
|
||||
showHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function showHelp() {
|
||||
console.log(
|
||||
boxen(
|
||||
`Task Master E2E Test Runner
|
||||
|
||||
Usage: node run-e2e-tests.js [options]
|
||||
|
||||
Options:
|
||||
--skip-verification Skip fallback verification tests
|
||||
--analyze-log [file] Analyze an existing log file
|
||||
--sequential Run tests sequentially instead of in parallel
|
||||
--groups <g1,g2> Run only specific test groups
|
||||
--help Show this help message
|
||||
|
||||
Test Groups: ${Object.keys(testGroups).join(', ')}`,
|
||||
{ padding: 1, margin: 1, borderStyle: 'round' }
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Generate timestamp for test run
|
||||
function generateTimestamp() {
|
||||
const now = new Date();
|
||||
const year = now.getFullYear();
|
||||
const month = String(now.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(now.getDate()).padStart(2, '0');
|
||||
const hours = String(now.getHours()).padStart(2, '0');
|
||||
const minutes = String(now.getMinutes()).padStart(2, '0');
|
||||
const seconds = String(now.getSeconds()).padStart(2, '0');
|
||||
|
||||
return `${year}${month}${day}_${hours}${minutes}${seconds}`;
|
||||
}
|
||||
|
||||
// Analyze existing log file
|
||||
async function analyzeLogFile(logFile, logger) {
|
||||
console.log(chalk.cyan('\n📊 Running LLM Analysis on log file...\n'));
|
||||
|
||||
const analyzer = new LLMAnalyzer(testConfig, logger);
|
||||
const analysis = await analyzer.analyzeLog(logFile);
|
||||
|
||||
if (analysis) {
|
||||
const report = analyzer.formatReport(analysis);
|
||||
displayAnalysisReport(report);
|
||||
} else {
|
||||
console.log(chalk.red('Failed to analyze log file'));
|
||||
}
|
||||
}
|
||||
|
||||
// Display analysis report
|
||||
function displayAnalysisReport(report) {
|
||||
console.log(
|
||||
boxen(chalk.cyan.bold(`${report.title}\n${report.timestamp}`), {
|
||||
padding: 1,
|
||||
borderStyle: 'double',
|
||||
borderColor: 'cyan'
|
||||
})
|
||||
);
|
||||
|
||||
// Status
|
||||
const statusColor =
|
||||
report.status === 'Success'
|
||||
? chalk.green
|
||||
: report.status === 'Warning'
|
||||
? chalk.yellow
|
||||
: chalk.red;
|
||||
console.log(
|
||||
boxen(`Overall Status: ${statusColor.bold(report.status)}`, {
|
||||
padding: { left: 1, right: 1 },
|
||||
borderColor: 'blue'
|
||||
})
|
||||
);
|
||||
|
||||
// Summary points
|
||||
if (report.summary && report.summary.length > 0) {
|
||||
console.log(chalk.blue.bold('\n📋 Summary Points:'));
|
||||
report.summary.forEach((point) => {
|
||||
console.log(chalk.white(` - ${point}`));
|
||||
});
|
||||
}
|
||||
|
||||
// Provider comparison
|
||||
if (report.providerComparison) {
|
||||
console.log(chalk.magenta.bold('\n🔄 Provider Comparison:'));
|
||||
const comp = report.providerComparison;
|
||||
|
||||
if (comp.provider_results) {
|
||||
Object.entries(comp.provider_results).forEach(([provider, result]) => {
|
||||
const status =
|
||||
result.status === 'Success' ? chalk.green('✅') : chalk.red('❌');
|
||||
console.log(` ${status} ${provider}: ${result.notes || 'N/A'}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Issues
|
||||
if (report.issues && report.issues.length > 0) {
|
||||
console.log(chalk.red.bold('\n🚨 Detected Issues:'));
|
||||
report.issues.forEach((issue, index) => {
|
||||
const severity =
|
||||
issue.severity === 'Error'
|
||||
? chalk.red('❌')
|
||||
: issue.severity === 'Warning'
|
||||
? chalk.yellow('⚠️')
|
||||
: 'ℹ️';
|
||||
console.log(
|
||||
boxen(
|
||||
`${severity} Issue ${index + 1}: [${issue.severity}]\n${issue.description}`,
|
||||
{ padding: 1, margin: { bottom: 1 }, borderColor: 'red' }
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Main test execution
|
||||
async function runTests(options) {
|
||||
const timestamp = generateTimestamp();
|
||||
const logger = new TestLogger(testConfig.paths.logDir, timestamp);
|
||||
const helpers = new TestHelpers(logger);
|
||||
|
||||
console.log(
|
||||
boxen(
|
||||
chalk.cyan.bold(
|
||||
`Task Master E2E Test Suite\n${new Date().toISOString()}`
|
||||
),
|
||||
{ padding: 1, borderStyle: 'double', borderColor: 'cyan' }
|
||||
)
|
||||
);
|
||||
|
||||
logger.info('Starting E2E test suite');
|
||||
logger.info(`Test configuration: ${JSON.stringify(options)}`);
|
||||
|
||||
// Update config based on options
|
||||
if (options.skipVerification) {
|
||||
testConfig.settings.runVerificationTest = false;
|
||||
}
|
||||
|
||||
let results;
|
||||
let testDir;
|
||||
|
||||
try {
|
||||
// Check dependencies
|
||||
logger.step('Checking dependencies');
|
||||
const deps = ['jq', 'bc'];
|
||||
for (const dep of deps) {
|
||||
const { exitCode } = await helpers.executeCommand('which', [dep]);
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(
|
||||
`Required dependency '${dep}' not found. Please install it.`
|
||||
);
|
||||
}
|
||||
}
|
||||
logger.success('All dependencies found');
|
||||
|
||||
// Determine which test groups to run
|
||||
const groupsToRun = options.groups || Object.keys(testGroups);
|
||||
const testsToRun = {};
|
||||
|
||||
groupsToRun.forEach((group) => {
|
||||
if (testGroups[group]) {
|
||||
testsToRun[group] = testGroups[group];
|
||||
} else {
|
||||
logger.warning(`Unknown test group: ${group}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (Object.keys(testsToRun).length === 0) {
|
||||
throw new Error('No valid test groups to run');
|
||||
}
|
||||
|
||||
// Check if we need to run setup (either explicitly requested or needed for other tests)
|
||||
const needsSetup = testsToRun.setup || (!testDir && Object.keys(testsToRun).length > 0);
|
||||
|
||||
if (needsSetup) {
|
||||
// Always run setup if we need a test directory
|
||||
if (!testsToRun.setup) {
|
||||
logger.info('No test directory available, running setup automatically');
|
||||
}
|
||||
|
||||
logger.step('Running setup tests');
|
||||
const setupRunner = new SequentialTestRunner(logger, helpers);
|
||||
const setupResults = await setupRunner.runTests(['setup'], {});
|
||||
|
||||
if (setupResults.tests.setup && setupResults.tests.setup.testDir) {
|
||||
testDir = setupResults.tests.setup.testDir;
|
||||
logger.info(`Test directory created: ${testDir}`);
|
||||
} else {
|
||||
throw new Error('Setup failed to create test directory');
|
||||
}
|
||||
|
||||
// Remove setup from remaining tests if it was explicitly requested
|
||||
if (testsToRun.setup) {
|
||||
delete testsToRun.setup;
|
||||
}
|
||||
}
|
||||
|
||||
// Run remaining tests
|
||||
if (Object.keys(testsToRun).length > 0) {
|
||||
const context = { testDir, config: testConfig };
|
||||
|
||||
if (options.parallel) {
|
||||
logger.info('Running tests in parallel mode');
|
||||
const parallelRunner = new ParallelTestRunner(logger);
|
||||
|
||||
try {
|
||||
results = await parallelRunner.runTestGroups(testsToRun, context);
|
||||
} finally {
|
||||
await parallelRunner.terminate();
|
||||
}
|
||||
} else {
|
||||
logger.info('Running tests in sequential mode');
|
||||
const sequentialRunner = new SequentialTestRunner(logger, helpers);
|
||||
|
||||
// Flatten test groups for sequential execution
|
||||
const allTests = Object.values(testsToRun).flat();
|
||||
results = await sequentialRunner.runTests(allTests, context);
|
||||
}
|
||||
}
|
||||
|
||||
// Final summary
|
||||
logger.flush();
|
||||
const summary = logger.getSummary();
|
||||
|
||||
displayTestSummary(summary, results);
|
||||
|
||||
// Run LLM analysis if enabled
|
||||
if (testConfig.llmAnalysis.enabled && !options.skipVerification) {
|
||||
await analyzeLogFile(summary.logFile, logger);
|
||||
}
|
||||
|
||||
// Exit with appropriate code
|
||||
const exitCode = results && results.overall === 'passed' ? 0 : 1;
|
||||
process.exit(exitCode);
|
||||
} catch (error) {
|
||||
logger.error(`Fatal error: ${error.message}`);
|
||||
logger.flush();
|
||||
|
||||
console.log(chalk.red.bold('\n❌ Test execution failed'));
|
||||
console.log(chalk.red(error.stack));
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Display test summary
|
||||
function displayTestSummary(summary, results) {
|
||||
console.log(
|
||||
boxen(chalk.cyan.bold('E2E Test Summary'), {
|
||||
padding: 1,
|
||||
margin: { top: 1 },
|
||||
borderStyle: 'round',
|
||||
borderColor: 'cyan'
|
||||
})
|
||||
);
|
||||
|
||||
console.log(chalk.white(`📁 Log File: ${summary.logFile}`));
|
||||
console.log(chalk.white(`⏱️ Duration: ${summary.duration}`));
|
||||
console.log(chalk.white(`📊 Total Steps: ${summary.totalSteps}`));
|
||||
console.log(chalk.green(`✅ Successes: ${summary.successCount}`));
|
||||
|
||||
if (summary.errorCount > 0) {
|
||||
console.log(chalk.red(`❌ Errors: ${summary.errorCount}`));
|
||||
}
|
||||
|
||||
console.log(chalk.yellow(`💰 Total Cost: $${summary.totalCost} USD`));
|
||||
|
||||
if (results) {
|
||||
const status =
|
||||
results.overall === 'passed'
|
||||
? chalk.green.bold('✅ PASSED')
|
||||
: chalk.red.bold('❌ FAILED');
|
||||
|
||||
console.log(
|
||||
boxen(`Overall Result: ${status}`, {
|
||||
padding: 1,
|
||||
margin: { top: 1 },
|
||||
borderColor: results.overall === 'passed' ? 'green' : 'red'
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Main entry point
|
||||
async function main() {
|
||||
const options = parseArgs();
|
||||
|
||||
if (options.analyzeLog) {
|
||||
// Analysis mode
|
||||
const logFile = options.logFile || (await findLatestLog());
|
||||
const logger = new TestLogger(testConfig.paths.logDir, 'analysis');
|
||||
|
||||
if (!existsSync(logFile)) {
|
||||
console.error(chalk.red(`Log file not found: ${logFile}`));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
await analyzeLogFile(logFile, logger);
|
||||
} else {
|
||||
// Test execution mode
|
||||
await runTests(options);
|
||||
}
|
||||
}
|
||||
|
||||
// Find the latest log file
|
||||
async function findLatestLog() {
|
||||
const { readdir } = await import('fs/promises');
|
||||
const files = await readdir(testConfig.paths.logDir);
|
||||
const logFiles = files
|
||||
.filter((f) => f.startsWith('e2e_run_') && f.endsWith('.log'))
|
||||
.sort()
|
||||
.reverse();
|
||||
|
||||
if (logFiles.length === 0) {
|
||||
throw new Error('No log files found');
|
||||
}
|
||||
|
||||
return join(testConfig.paths.logDir, logFiles[0]);
|
||||
}
|
||||
|
||||
// Run the main function
|
||||
main().catch((error) => {
|
||||
console.error(chalk.red('Unexpected error:'), error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user