diff --git a/data/nodes.db b/data/nodes.db index d7a468b..98af90b 100644 Binary files a/data/nodes.db and b/data/nodes.db differ diff --git a/package.json b/package.json index 11e97f9..b36b325 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "update:n8n:check": "node scripts/update-n8n-deps.js --dry-run", "fetch:templates": "node dist/scripts/fetch-templates.js", "fetch:templates:robust": "node dist/scripts/fetch-templates-robust.js", + "prebuild:fts5": "npx tsx scripts/prebuild-fts5.ts", "test:templates": "node dist/scripts/test-templates.js", "test:workflow-validation": "node dist/scripts/test-workflow-validation.js", "test:template-validation": "node dist/scripts/test-template-validation.js", @@ -40,6 +41,9 @@ "test:workflow-diff": "node dist/scripts/test-workflow-diff.js", "test:transactional-diff": "node dist/scripts/test-transactional-diff.js", "test:tools-documentation": "node dist/scripts/test-tools-documentation.js", + "test:search-improvements": "node dist/scripts/test-search-improvements.js", + "test:fts5-search": "node dist/scripts/test-fts5-search.js", + "migrate:fts5": "node dist/scripts/migrate-nodes-fts.js", "test:mcp:update-partial": "node dist/scripts/test-mcp-n8n-update-partial.js", "test:update-partial:debug": "node dist/scripts/test-update-partial-debug.js", "test:auth-logging": "tsx scripts/test-auth-logging.ts", diff --git a/scripts/prebuild-fts5.ts b/scripts/prebuild-fts5.ts new file mode 100644 index 0000000..dc6b70b --- /dev/null +++ b/scripts/prebuild-fts5.ts @@ -0,0 +1,111 @@ +#!/usr/bin/env npx tsx +/** + * Pre-build FTS5 indexes for the database + * This ensures FTS5 tables are created before the database is deployed to Docker + */ +import { createDatabaseAdapter } from '../src/database/database-adapter'; +import { logger } from '../src/utils/logger'; +import * as fs from 'fs'; + +async function prebuildFTS5() { + console.log('๐Ÿ” Pre-building FTS5 indexes...\n'); + + const dbPath = './data/nodes.db'; + + if (!fs.existsSync(dbPath)) { + console.error('โŒ Database not found at', dbPath); + console.error(' Please run npm run rebuild first'); + process.exit(1); + } + + const db = await createDatabaseAdapter(dbPath); + + // Check FTS5 support + const hasFTS5 = db.checkFTS5Support(); + + if (!hasFTS5) { + console.log('โ„น๏ธ FTS5 not supported in this SQLite build'); + console.log(' Skipping FTS5 pre-build'); + db.close(); + return; + } + + console.log('โœ… FTS5 is supported'); + + try { + // Create FTS5 virtual table for templates + console.log('\n๐Ÿ“‹ Creating FTS5 table for templates...'); + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5( + name, description, content=templates + ); + `); + + // Create triggers to keep FTS5 in sync + console.log('๐Ÿ”— Creating synchronization triggers...'); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates BEGIN + INSERT INTO templates_fts(rowid, name, description) + VALUES (new.id, new.name, new.description); + END; + `); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates BEGIN + UPDATE templates_fts SET name = new.name, description = new.description + WHERE rowid = new.id; + END; + `); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates BEGIN + DELETE FROM templates_fts WHERE rowid = old.id; + END; + `); + + // Rebuild FTS5 index from existing data + console.log('๐Ÿ”„ Rebuilding FTS5 index from existing templates...'); + + // Clear existing FTS data + db.exec('DELETE FROM templates_fts'); + + // Repopulate from templates table + db.exec(` + INSERT INTO templates_fts(rowid, name, description) + SELECT id, name, description FROM templates + `); + + // Get counts + const templateCount = db.prepare('SELECT COUNT(*) as count FROM templates').get() as { count: number }; + const ftsCount = db.prepare('SELECT COUNT(*) as count FROM templates_fts').get() as { count: number }; + + console.log(`\nโœ… FTS5 pre-build complete!`); + console.log(` Templates: ${templateCount.count}`); + console.log(` FTS5 entries: ${ftsCount.count}`); + + // Test FTS5 search + console.log('\n๐Ÿงช Testing FTS5 search...'); + const testResults = db.prepare(` + SELECT COUNT(*) as count FROM templates t + JOIN templates_fts ON t.id = templates_fts.rowid + WHERE templates_fts MATCH 'webhook' + `).get() as { count: number }; + + console.log(` Found ${testResults.count} templates matching "webhook"`); + + } catch (error) { + console.error('โŒ Error pre-building FTS5:', error); + process.exit(1); + } + + db.close(); + console.log('\nโœ… Database is ready for Docker deployment!'); +} + +// Run if called directly +if (require.main === module) { + prebuildFTS5().catch(console.error); +} + +export { prebuildFTS5 }; \ No newline at end of file diff --git a/src/scripts/fetch-templates.ts b/src/scripts/fetch-templates.ts index 9ec5536..a908d18 100644 --- a/src/scripts/fetch-templates.ts +++ b/src/scripts/fetch-templates.ts @@ -29,6 +29,49 @@ async function fetchTemplates() { const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8'); db.exec(schema); + // Pre-create FTS5 tables if supported + const hasFTS5 = db.checkFTS5Support(); + if (hasFTS5) { + console.log('๐Ÿ” Creating FTS5 tables for template search...'); + try { + // Create FTS5 virtual table + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5( + name, description, content=templates + ); + `); + + // Create triggers to keep FTS5 in sync + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates BEGIN + INSERT INTO templates_fts(rowid, name, description) + VALUES (new.id, new.name, new.description); + END; + `); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates BEGIN + UPDATE templates_fts SET name = new.name, description = new.description + WHERE rowid = new.id; + END; + `); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates BEGIN + DELETE FROM templates_fts WHERE rowid = old.id; + END; + `); + + console.log('โœ… FTS5 tables created successfully\n'); + } catch (error) { + console.log('โš ๏ธ Failed to create FTS5 tables:', error); + console.log(' Template search will use LIKE fallback\n'); + } + } else { + console.log('โ„น๏ธ FTS5 not supported in this SQLite build'); + console.log(' Template search will use LIKE queries\n'); + } + // Create service const service = new TemplateService(db); diff --git a/src/templates/template-repository.ts b/src/templates/template-repository.ts index 8caa9b9..5e59d62 100644 --- a/src/templates/template-repository.ts +++ b/src/templates/template-repository.ts @@ -38,37 +38,62 @@ export class TemplateRepository { if (this.hasFTS5Support) { try { - // Create FTS5 virtual table - this.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5( - name, description, content=templates - ); - `); + // Check if FTS5 table already exists + const ftsExists = this.db.prepare(` + SELECT name FROM sqlite_master + WHERE type='table' AND name='templates_fts' + `).get() as { name: string } | undefined; - // Create triggers to keep FTS5 in sync - this.db.exec(` - CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates BEGIN - INSERT INTO templates_fts(rowid, name, description) - VALUES (new.id, new.name, new.description); - END; - `); - - this.db.exec(` - CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates BEGIN - UPDATE templates_fts SET name = new.name, description = new.description - WHERE rowid = new.id; - END; - `); - - this.db.exec(` - CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates BEGIN - DELETE FROM templates_fts WHERE rowid = old.id; - END; - `); - - logger.info('FTS5 support enabled for template search'); - } catch (error) { - logger.warn('Failed to initialize FTS5 for templates:', error); + if (ftsExists) { + logger.info('FTS5 table already exists for templates'); + + // Verify FTS5 is working by doing a test query + try { + const testCount = this.db.prepare('SELECT COUNT(*) as count FROM templates_fts').get() as { count: number }; + logger.info(`FTS5 enabled with ${testCount.count} indexed entries`); + } catch (testError) { + logger.warn('FTS5 table exists but query failed:', testError); + this.hasFTS5Support = false; + return; + } + } else { + // Create FTS5 virtual table + logger.info('Creating FTS5 virtual table for templates...'); + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5( + name, description, content=templates + ); + `); + + // Create triggers to keep FTS5 in sync + this.db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates BEGIN + INSERT INTO templates_fts(rowid, name, description) + VALUES (new.id, new.name, new.description); + END; + `); + + this.db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates BEGIN + UPDATE templates_fts SET name = new.name, description = new.description + WHERE rowid = new.id; + END; + `); + + this.db.exec(` + CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates BEGIN + DELETE FROM templates_fts WHERE rowid = old.id; + END; + `); + + logger.info('FTS5 support enabled for template search'); + } + } catch (error: any) { + logger.warn('Failed to initialize FTS5 for templates:', { + message: error.message, + code: error.code, + stack: error.stack + }); this.hasFTS5Support = false; } } else { @@ -158,25 +183,36 @@ export class TemplateRepository { * Search templates by name or description */ searchTemplates(query: string, limit: number = 20): StoredTemplate[] { + logger.debug(`Searching templates for: "${query}" (FTS5: ${this.hasFTS5Support})`); + // If FTS5 is not supported, go straight to LIKE search if (!this.hasFTS5Support) { + logger.debug('Using LIKE search (FTS5 not available)'); return this.searchTemplatesLIKE(query, limit); } try { // Use FTS for search const ftsQuery = query.split(' ').map(term => `"${term}"`).join(' OR '); + logger.debug(`FTS5 query: ${ftsQuery}`); - return this.db.prepare(` + const results = this.db.prepare(` SELECT t.* FROM templates t JOIN templates_fts ON t.id = templates_fts.rowid WHERE templates_fts MATCH ? ORDER BY rank, t.views DESC LIMIT ? `).all(ftsQuery, limit) as StoredTemplate[]; + + logger.debug(`FTS5 search returned ${results.length} results`); + return results; } catch (error: any) { // If FTS5 query fails, fallback to LIKE search - logger.warn('FTS5 template search failed, using LIKE fallback:', error.message); + logger.warn('FTS5 template search failed, using LIKE fallback:', { + message: error.message, + query: query, + ftsQuery: query.split(' ').map(term => `"${term}"`).join(' OR ') + }); return this.searchTemplatesLIKE(query, limit); } } @@ -186,13 +222,17 @@ export class TemplateRepository { */ private searchTemplatesLIKE(query: string, limit: number = 20): StoredTemplate[] { const likeQuery = `%${query}%`; + logger.debug(`Using LIKE search with pattern: ${likeQuery}`); - return this.db.prepare(` + const results = this.db.prepare(` SELECT * FROM templates WHERE name LIKE ? OR description LIKE ? ORDER BY views DESC, created_at DESC LIMIT ? `).all(likeQuery, likeQuery, limit) as StoredTemplate[]; + + logger.debug(`LIKE search returned ${results.length} results`); + return results; } /**