diff --git a/src/templates/template-repository.ts b/src/templates/template-repository.ts index 696e449..c867800 100644 --- a/src/templates/template-repository.ts +++ b/src/templates/template-repository.ts @@ -625,22 +625,23 @@ export class TemplateRepository { return { total, withMetadata, withoutMetadata, outdated }; } - + /** - * Search templates by metadata fields + * Build WHERE conditions for metadata filtering + * @private + * @returns Object containing SQL conditions array and parameter values array */ - searchTemplatesByMetadata(filters: { + private buildMetadataFilterConditions(filters: { category?: string; complexity?: 'simple' | 'medium' | 'complex'; maxSetupMinutes?: number; minSetupMinutes?: number; requiredService?: string; targetAudience?: string; - }, limit: number = 20, offset: number = 0): StoredTemplate[] { + }): { conditions: string[], params: any[] } { const conditions: string[] = ['metadata_json IS NOT NULL']; const params: any[] = []; - // Build WHERE conditions based on filters with proper parameterization if (filters.category !== undefined) { // Use parameterized LIKE with JSON array search - safe from injection conditions.push("json_extract(metadata_json, '$.categories') LIKE '%' || ? || '%'"); @@ -680,34 +681,86 @@ export class TemplateRepository { params.push(sanitizedAudience); } + return { conditions, params }; + } + + /** + * Search templates by metadata fields + */ + searchTemplatesByMetadata(filters: { + category?: string; + complexity?: 'simple' | 'medium' | 'complex'; + maxSetupMinutes?: number; + minSetupMinutes?: number; + requiredService?: string; + targetAudience?: string; + }, limit: number = 20, offset: number = 0): StoredTemplate[] { + const startTime = Date.now(); + + // Build WHERE conditions using shared helper + const { conditions, params } = this.buildMetadataFilterConditions(filters); + // Performance optimization: Use two-phase query to avoid loading large compressed workflows // during metadata filtering. This prevents timeout when no filters are provided. // Phase 1: Get IDs only with metadata filtering (fast - no workflow data) + // Add id to ORDER BY to ensure stable ordering const idsQuery = ` SELECT id FROM templates WHERE ${conditions.join(' AND ')} - ORDER BY views DESC, created_at DESC + ORDER BY views DESC, created_at DESC, id ASC LIMIT ? OFFSET ? `; params.push(limit, offset); const ids = this.db.prepare(idsQuery).all(...params) as { id: number }[]; + const phase1Time = Date.now() - startTime; + if (ids.length === 0) { - logger.debug('Metadata search found 0 results', { filters }); + logger.debug('Metadata search found 0 results', { filters, phase1Ms: phase1Time }); return []; } - // Phase 2: Fetch full records only for matching IDs (only decompress needed rows) - const placeholders = ids.map(() => '?').join(','); - const fullQuery = ` - SELECT * FROM templates - WHERE id IN (${placeholders}) - ORDER BY views DESC, created_at DESC - `; - const results = this.db.prepare(fullQuery).all(...ids.map(r => r.id)) as StoredTemplate[]; + // Defensive validation: ensure all IDs are valid positive integers + const idValues = ids.map(r => r.id).filter(id => typeof id === 'number' && id > 0 && Number.isInteger(id)); + + if (idValues.length === 0) { + logger.warn('No valid IDs after filtering', { filters, originalCount: ids.length }); + return []; + } + + if (idValues.length !== ids.length) { + logger.warn('Some IDs were filtered out as invalid', { + original: ids.length, + valid: idValues.length, + filtered: ids.length - idValues.length + }); + } + + // Phase 2: Fetch full records preserving exact order from Phase 1 + // Use CTE with VALUES to maintain ordering without depending on SQLite's IN clause behavior + const phase2Start = Date.now(); + const orderedQuery = ` + WITH ordered_ids(id, sort_order) AS ( + VALUES ${idValues.map((id, idx) => `(${id}, ${idx})`).join(', ')} + ) + SELECT t.* FROM templates t + INNER JOIN ordered_ids o ON t.id = o.id + ORDER BY o.sort_order + `; + + const results = this.db.prepare(orderedQuery).all() as StoredTemplate[]; + const phase2Time = Date.now() - phase2Start; + + logger.debug(`Metadata search found ${results.length} results`, { + filters, + count: results.length, + phase1Ms: phase1Time, + phase2Ms: phase2Time, + totalMs: Date.now() - startTime, + optimization: 'two-phase-with-ordering' + }); - logger.debug(`Metadata search found ${results.length} results`, { filters, count: results.length }); return results.map(t => this.decompressWorkflow(t)); } @@ -722,48 +775,12 @@ export class TemplateRepository { requiredService?: string; targetAudience?: string; }): number { - const conditions: string[] = ['metadata_json IS NOT NULL']; - const params: any[] = []; - - if (filters.category !== undefined) { - // Use parameterized LIKE with JSON array search - safe from injection - conditions.push("json_extract(metadata_json, '$.categories') LIKE '%' || ? || '%'"); - const sanitizedCategory = JSON.stringify(filters.category).slice(1, -1); - params.push(sanitizedCategory); - } - - if (filters.complexity) { - conditions.push("json_extract(metadata_json, '$.complexity') = ?"); - params.push(filters.complexity); - } - - if (filters.maxSetupMinutes !== undefined) { - conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) <= ?"); - params.push(filters.maxSetupMinutes); - } - - if (filters.minSetupMinutes !== undefined) { - conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) >= ?"); - params.push(filters.minSetupMinutes); - } - - if (filters.requiredService !== undefined) { - // Use parameterized LIKE with JSON array search - safe from injection - conditions.push("json_extract(metadata_json, '$.required_services') LIKE '%' || ? || '%'"); - const sanitizedService = JSON.stringify(filters.requiredService).slice(1, -1); - params.push(sanitizedService); - } - - if (filters.targetAudience !== undefined) { - // Use parameterized LIKE with JSON array search - safe from injection - conditions.push("json_extract(metadata_json, '$.target_audience') LIKE '%' || ? || '%'"); - const sanitizedAudience = JSON.stringify(filters.targetAudience).slice(1, -1); - params.push(sanitizedAudience); - } - + // Build WHERE conditions using shared helper + const { conditions, params } = this.buildMetadataFilterConditions(filters); + const query = `SELECT COUNT(*) as count FROM templates WHERE ${conditions.join(' AND ')}`; const result = this.db.prepare(query).get(...params) as { count: number }; - + return result.count; } diff --git a/tests/integration/database/template-repository.test.ts b/tests/integration/database/template-repository.test.ts index bfac754..88b4849 100644 --- a/tests/integration/database/template-repository.test.ts +++ b/tests/integration/database/template-repository.test.ts @@ -643,6 +643,202 @@ describe('TemplateRepository Integration Tests', () => { }); }); }); + + describe('searchTemplatesByMetadata - Two-Phase Optimization', () => { + it('should use two-phase query pattern for performance', () => { + // Setup: Create templates with metadata + const templates = [ + { id: 1, complexity: 'simple', category: 'automation' }, + { id: 2, complexity: 'medium', category: 'integration' }, + { id: 3, complexity: 'simple', category: 'automation' }, + { id: 4, complexity: 'complex', category: 'data-processing' } + ]; + + templates.forEach(({ id, complexity, category }) => { + const template = createTemplateWorkflow({ id, name: `Template ${id}` }); + const detail = createTemplateDetail({ + id, + workflow: { + id: id.toString(), + name: `Template ${id}`, + nodes: [], + connections: {}, + settings: {} + } + }); + + repository.saveTemplate(template, detail); + + // Add metadata + const metadata = { + categories: [category], + complexity, + use_cases: ['test'], + estimated_setup_minutes: 15, + required_services: [], + key_features: ['test'], + target_audience: ['developers'] + }; + + db.prepare(` + UPDATE templates + SET metadata_json = ?, + metadata_generated_at = datetime('now') + WHERE workflow_id = ? + `).run(JSON.stringify(metadata), id); + }); + + // Test: Search with filter should return matching templates + const results = repository.searchTemplatesByMetadata({ complexity: 'simple' }, 10, 0); + + // Verify results + expect(results).toHaveLength(2); + expect(results[0].workflow_id).toBe(1); // Ordered by views DESC, then created_at DESC, then id ASC + expect(results[1].workflow_id).toBe(3); + }); + + it('should preserve exact ordering from Phase 1', () => { + // Setup: Create templates with different view counts + const templates = [ + { id: 1, views: 100 }, + { id: 2, views: 500 }, + { id: 3, views: 300 }, + { id: 4, views: 500 }, // Same views as id:2, should be ordered by id + { id: 5, views: 200 } + ]; + + templates.forEach(({ id, views }) => { + const template = createTemplateWorkflow({ id, name: `Template ${id}`, totalViews: views }); + const detail = createTemplateDetail({ + id, + views, + workflow: { + id: id.toString(), + name: `Template ${id}`, + nodes: [], + connections: {}, + settings: {} + } + }); + + repository.saveTemplate(template, detail); + + // Update views in database to match our test data + db.prepare(`UPDATE templates SET views = ? WHERE workflow_id = ?`).run(views, id); + + // Add metadata + const metadata = { + categories: ['test'], + complexity: 'medium', + use_cases: ['test'], + estimated_setup_minutes: 15, + required_services: [], + key_features: ['test'], + target_audience: ['developers'] + }; + + db.prepare(` + UPDATE templates + SET metadata_json = ?, + metadata_generated_at = datetime('now') + WHERE workflow_id = ? + `).run(JSON.stringify(metadata), id); + }); + + // Test: Search should return templates in correct order + const results = repository.searchTemplatesByMetadata({ complexity: 'medium' }, 10, 0); + + // Verify ordering: 500 views (id 2), 500 views (id 4), 300 views, 200 views, 100 views + expect(results).toHaveLength(5); + expect(results[0].workflow_id).toBe(2); // 500 views, lower id + expect(results[1].workflow_id).toBe(4); // 500 views, higher id + expect(results[2].workflow_id).toBe(3); // 300 views + expect(results[3].workflow_id).toBe(5); // 200 views + expect(results[4].workflow_id).toBe(1); // 100 views + }); + + it('should handle empty results efficiently', () => { + // Setup: Create templates without the searched complexity + const template = createTemplateWorkflow({ id: 1 }); + const detail = createTemplateDetail({ + id: 1, + workflow: { + id: '1', + name: 'Template 1', + nodes: [], + connections: {}, + settings: {} + } + }); + + repository.saveTemplate(template, detail); + + const metadata = { + categories: ['test'], + complexity: 'simple', + use_cases: ['test'], + estimated_setup_minutes: 15, + required_services: [], + key_features: ['test'], + target_audience: ['developers'] + }; + + db.prepare(` + UPDATE templates + SET metadata_json = ?, + metadata_generated_at = datetime('now') + WHERE workflow_id = 1 + `).run(JSON.stringify(metadata)); + + // Test: Search for non-existent complexity + const results = repository.searchTemplatesByMetadata({ complexity: 'complex' }, 10, 0); + + // Verify: Should return empty array without errors + expect(results).toHaveLength(0); + }); + + it('should validate IDs defensively', () => { + // This test ensures the defensive ID validation works + // Setup: Create a template + const template = createTemplateWorkflow({ id: 1 }); + const detail = createTemplateDetail({ + id: 1, + workflow: { + id: '1', + name: 'Template 1', + nodes: [], + connections: {}, + settings: {} + } + }); + + repository.saveTemplate(template, detail); + + const metadata = { + categories: ['test'], + complexity: 'simple', + use_cases: ['test'], + estimated_setup_minutes: 15, + required_services: [], + key_features: ['test'], + target_audience: ['developers'] + }; + + db.prepare(` + UPDATE templates + SET metadata_json = ?, + metadata_generated_at = datetime('now') + WHERE workflow_id = 1 + `).run(JSON.stringify(metadata)); + + // Test: Normal search should work + const results = repository.searchTemplatesByMetadata({ complexity: 'simple' }, 10, 0); + + // Verify: Should return the template + expect(results).toHaveLength(1); + expect(results[0].workflow_id).toBe(1); + }); + }); }); // Helper functions