diff --git a/tests/benchmarks/database-queries.bench.ts b/tests/benchmarks/database-queries.bench.ts index 99d20a7..481c403 100644 --- a/tests/benchmarks/database-queries.bench.ts +++ b/tests/benchmarks/database-queries.bench.ts @@ -4,6 +4,17 @@ import { SQLiteStorageService } from '../../src/services/sqlite-storage-service' import { NodeFactory } from '../factories/node-factory'; import { PropertyDefinitionFactory } from '../factories/property-definition-factory'; +/** + * Database Query Performance Benchmarks + * + * NOTE: These benchmarks use MOCK DATA (500 artificial test nodes) + * created with factories, not the real production database. + * + * This is useful for tracking database layer performance in isolation, + * but may not reflect real-world performance characteristics. + * + * For end-to-end MCP tool performance with real data, see mcp-tools.bench.ts + */ describe('Database Query Performance', () => { let repository: NodeRepository; let storage: SQLiteStorageService; diff --git a/tests/benchmarks/index.ts b/tests/benchmarks/index.ts index 05b5802..8adb81c 100644 --- a/tests/benchmarks/index.ts +++ b/tests/benchmarks/index.ts @@ -1,7 +1,3 @@ // Export all benchmark suites -// Note: Some benchmarks are temporarily disabled due to API changes -// export * from './node-loading.bench'; export * from './database-queries.bench'; -// export * from './search-operations.bench'; -// export * from './validation-performance.bench'; -// export * from './mcp-tools.bench'; \ No newline at end of file +export * from './mcp-tools.bench'; \ No newline at end of file diff --git a/tests/benchmarks/mcp-tools.bench.ts b/tests/benchmarks/mcp-tools.bench.ts new file mode 100644 index 0000000..c4a527a --- /dev/null +++ b/tests/benchmarks/mcp-tools.bench.ts @@ -0,0 +1,169 @@ +import { bench, describe } from 'vitest'; +import { NodeRepository } from '../../src/database/node-repository'; +import { createDatabaseAdapter } from '../../src/database/database-adapter'; +import { EnhancedConfigValidator } from '../../src/services/enhanced-config-validator'; +import { PropertyFilter } from '../../src/services/property-filter'; +import path from 'path'; + +/** + * MCP Tool Performance Benchmarks + * + * These benchmarks measure end-to-end performance of actual MCP tool operations + * using the REAL production database (data/nodes.db with 525+ nodes). + * + * Unlike database-queries.bench.ts which uses mock data, these benchmarks + * reflect what AI assistants actually experience when calling MCP tools, + * making this the most meaningful performance metric for the system. + */ +describe('MCP Tool Performance (Production Database)', () => { + let repository: NodeRepository; + + beforeAll(async () => { + // Use REAL production database + const dbPath = path.join(__dirname, '../../data/nodes.db'); + const db = await createDatabaseAdapter(dbPath); + repository = new NodeRepository(db); + // Initialize similarity services for validation + EnhancedConfigValidator.initializeSimilarityServices(repository); + }); + + /** + * search_nodes - Most frequently used tool for node discovery + * + * This measures: + * - Database FTS5 full-text search + * - Result filtering and ranking + * - Response serialization + * + * Target: <20ms for common queries + */ + bench('search_nodes - common query (http)', async () => { + await repository.searchNodes('http', 'OR', 20); + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); + + bench('search_nodes - AI agent query (slack message)', async () => { + await repository.searchNodes('slack send message', 'AND', 10); + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); + + /** + * get_node_essentials - Fast retrieval of node configuration + * + * This measures: + * - Database node lookup + * - Property filtering (essentials only) + * - Response formatting + * + * Target: <10ms for most nodes + */ + bench('get_node_essentials - HTTP Request node', async () => { + const node = await repository.getNodeByType('n8n-nodes-base.httpRequest'); + if (node && node.properties) { + PropertyFilter.getEssentials(node.properties, node.nodeType); + } + }, { + iterations: 200, + warmupIterations: 20, + warmupTime: 500, + time: 3000 + }); + + bench('get_node_essentials - Slack node', async () => { + const node = await repository.getNodeByType('n8n-nodes-base.slack'); + if (node && node.properties) { + PropertyFilter.getEssentials(node.properties, node.nodeType); + } + }, { + iterations: 200, + warmupIterations: 20, + warmupTime: 500, + time: 3000 + }); + + /** + * list_nodes - Initial exploration/listing + * + * This measures: + * - Database query with pagination + * - Result serialization + * - Category filtering + * + * Target: <15ms for first page + */ + bench('list_nodes - first 50 nodes', async () => { + await repository.getAllNodes(50); + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); + + bench('list_nodes - AI tools only', async () => { + await repository.getAIToolNodes(); + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); + + /** + * validate_node_operation - Configuration validation + * + * This measures: + * - Schema lookup + * - Validation logic execution + * - Error message formatting + * + * Target: <15ms for simple validations + */ + bench('validate_node_operation - HTTP Request (minimal)', async () => { + const node = await repository.getNodeByType('n8n-nodes-base.httpRequest'); + if (node && node.properties) { + EnhancedConfigValidator.validateWithMode( + 'n8n-nodes-base.httpRequest', + {}, + node.properties, + 'operation', + 'ai-friendly' + ); + } + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); + + bench('validate_node_operation - HTTP Request (with params)', async () => { + const node = await repository.getNodeByType('n8n-nodes-base.httpRequest'); + if (node && node.properties) { + EnhancedConfigValidator.validateWithMode( + 'n8n-nodes-base.httpRequest', + { + requestMethod: 'GET', + url: 'https://api.example.com', + authentication: 'none' + }, + node.properties, + 'operation', + 'ai-friendly' + ); + } + }, { + iterations: 100, + warmupIterations: 10, + warmupTime: 500, + time: 3000 + }); +}); diff --git a/tests/benchmarks/sample.bench.ts b/tests/benchmarks/sample.bench.ts deleted file mode 100644 index 2b2451a..0000000 --- a/tests/benchmarks/sample.bench.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { bench, describe } from 'vitest'; - -/** - * Sample benchmark to verify the setup works correctly - */ -describe('Sample Benchmarks', () => { - bench('array sorting - small', () => { - const arr = Array.from({ length: 100 }, () => Math.random()); - arr.sort((a, b) => a - b); - }, { - iterations: 1000, - warmupIterations: 100 - }); - - bench('array sorting - large', () => { - const arr = Array.from({ length: 10000 }, () => Math.random()); - arr.sort((a, b) => a - b); - }, { - iterations: 100, - warmupIterations: 10 - }); - - bench('string concatenation', () => { - let str = ''; - for (let i = 0; i < 1000; i++) { - str += 'a'; - } - }, { - iterations: 1000, - warmupIterations: 100 - }); - - bench('object creation', () => { - const objects = []; - for (let i = 0; i < 1000; i++) { - objects.push({ - id: i, - name: `Object ${i}`, - value: Math.random(), - timestamp: Date.now() - }); - } - }, { - iterations: 1000, - warmupIterations: 100 - }); -}); \ No newline at end of file