fix: add FTS5 search index to prevent 69% search failure rate (v2.18.5)

Fixes production search failures where 69% of user searches returned zero
results for critical nodes (webhook, merge, split batch) despite nodes
existing in database.

Root Cause:
- schema.sql missing nodes_fts FTS5 virtual table
- No validation to detect empty database or missing FTS5
- rebuild.ts used schema without search index
- Result: 9 of 13 searches failed in production

Changes:
1. Schema Updates (src/database/schema.sql):
   - Added nodes_fts FTS5 virtual table with full-text indexing
   - Added INSERT/UPDATE/DELETE triggers for auto-sync
   - Indexes: node_type, display_name, description, documentation, operations

2. Database Validation (src/scripts/rebuild.ts):
   - Added empty database detection (fails if zero nodes)
   - Added FTS5 existence and synchronization validation
   - Added searchability tests for critical nodes
   - Added minimum node count check (500+)

3. Runtime Health Checks (src/mcp/server.ts):
   - Database health validation on first access
   - Detects empty database with clear error
   - Detects missing FTS5 with actionable warning

4. Test Suite (53 new tests):
   - tests/integration/database/node-fts5-search.test.ts (14 tests)
   - tests/integration/database/empty-database.test.ts (14 tests)
   - tests/integration/ci/database-population.test.ts (25 tests)

5. Database Rebuild:
   - data/nodes.db rebuilt with FTS5 index
   - 535 nodes fully synchronized with FTS5

Impact:
- ✅ All critical searches now work (webhook, merge, split, code, http)
- ✅ FTS5 provides fast ranked search (< 100ms)
- ✅ Clear error messages if database empty
- ✅ CI validates committed database integrity
- ✅ Runtime health checks detect issues immediately

Performance:
- FTS5 search: < 100ms for typical queries
- LIKE fallback: < 500ms (unchanged, still functional)

Testing: LIKE search investigation revealed it was perfectly functional,
only failed because database was empty. No changes needed.

Related: Issue #296 Part 2 (Part 1: v2.18.4 fixed adapter bypass)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/database/schema.sql

@@ -25,6 +25,40 @@ CREATE INDEX IF NOT EXISTS idx_package ON nodes(package_name);
 CREATE INDEX IF NOT EXISTS idx_ai_tool ON nodes(is_ai_tool);
 CREATE INDEX IF NOT EXISTS idx_category ON nodes(category);
 
+-- FTS5 full-text search index for nodes
+CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
+  node_type,
+  display_name,
+  description,
+  documentation,
+  operations,
+  content=nodes,
+  content_rowid=rowid
+);
+
+-- Triggers to keep FTS5 in sync with nodes table
+CREATE TRIGGER IF NOT EXISTS nodes_fts_insert AFTER INSERT ON nodes
+BEGIN
+  INSERT INTO nodes_fts(rowid, node_type, display_name, description, documentation, operations)
+  VALUES (new.rowid, new.node_type, new.display_name, new.description, new.documentation, new.operations);
+END;
+
+CREATE TRIGGER IF NOT EXISTS nodes_fts_update AFTER UPDATE ON nodes
+BEGIN
+  UPDATE nodes_fts
+  SET node_type = new.node_type,
+      display_name = new.display_name,
+      description = new.description,
+      documentation = new.documentation,
+      operations = new.operations
+  WHERE rowid = new.rowid;
+END;
+
+CREATE TRIGGER IF NOT EXISTS nodes_fts_delete AFTER DELETE ON nodes
+BEGIN
+  DELETE FROM nodes_fts WHERE rowid = old.rowid;
+END;
+
 -- Templates table for n8n workflow templates
 CREATE TABLE IF NOT EXISTS templates (
   id INTEGER PRIMARY KEY,
@@ -108,5 +142,6 @@ FROM template_node_configs
 WHERE rank <= 5  -- Top 5 per node type
 ORDER BY node_type, rank;
 
--- Note: FTS5 tables are created conditionally at runtime if FTS5 is supported
--- See template-repository.ts initializeFTS5() method
+-- Note: Template FTS5 tables are created conditionally at runtime if FTS5 is supported
+-- See template-repository.ts initializeFTS5() method
+-- Node FTS5 table (nodes_fts) is created above during schema initialization

src/mcp/server.ts

@@ -201,6 +201,48 @@ export class N8NDocumentationMCPServer {
     if (!this.db || !this.repository) {
       throw new Error('Database not initialized');
     }
+
+    // Validate database health on first access
+    if (!this.dbHealthChecked) {
+      await this.validateDatabaseHealth();
+      this.dbHealthChecked = true;
+    }
+  }
+
+  private dbHealthChecked: boolean = false;
+
+  private async validateDatabaseHealth(): Promise<void> {
+    if (!this.db) return;
+
+    try {
+      // Check if nodes table has data
+      const nodeCount = this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as { count: number };
+
+      if (nodeCount.count === 0) {
+        logger.error('CRITICAL: Database is empty - no nodes found! Please run: npm run rebuild');
+        throw new Error('Database is empty. Run "npm run rebuild" to populate node data.');
+      }
+
+      // Check if FTS5 table exists
+      const ftsExists = this.db.prepare(`
+        SELECT name FROM sqlite_master
+        WHERE type='table' AND name='nodes_fts'
+      `).get();
+
+      if (!ftsExists) {
+        logger.warn('FTS5 table missing - search performance will be degraded. Please run: npm run rebuild');
+      } else {
+        const ftsCount = this.db.prepare('SELECT COUNT(*) as count FROM nodes_fts').get() as { count: number };
+        if (ftsCount.count === 0) {
+          logger.warn('FTS5 index is empty - search will not work properly. Please run: npm run rebuild');
+        }
+      }
+
+      logger.info(`Database health check passed: ${nodeCount.count} nodes loaded`);
+    } catch (error) {
+      logger.error('Database health check failed:', error);
+      throw error;
+    }
   }
 
   private setupHandlers(): void {

src/scripts/rebuild.ts

@@ -167,29 +167,81 @@ async function rebuild() {
 
 function validateDatabase(repository: NodeRepository): { passed: boolean; issues: string[] } {
   const issues = [];
-  
-  // Check critical nodes
-  const criticalNodes = ['nodes-base.httpRequest', 'nodes-base.code', 'nodes-base.webhook', 'nodes-base.slack'];
-  
-  for (const nodeType of criticalNodes) {
-    const node = repository.getNode(nodeType);
-    
-    if (!node) {
-      issues.push(`Critical node ${nodeType} not found`);
-      continue;
+
+  try {
+    const db = (repository as any).db;
+
+    // CRITICAL: Check if database has any nodes at all
+    const nodeCount = db.prepare('SELECT COUNT(*) as count FROM nodes').get() as { count: number };
+    if (nodeCount.count === 0) {
+      issues.push('CRITICAL: Database is empty - no nodes found! Rebuild failed or was interrupted.');
+      return { passed: false, issues };
     }
-    
-    if (node.properties.length === 0) {
-      issues.push(`Node ${nodeType} has no properties`);
+
+    // Check minimum expected node count (should have at least 500 nodes from both packages)
+    if (nodeCount.count < 500) {
+      issues.push(`WARNING: Only ${nodeCount.count} nodes found - expected at least 500 (both n8n packages)`);
     }
+
+    // Check critical nodes
+    const criticalNodes = ['nodes-base.httpRequest', 'nodes-base.code', 'nodes-base.webhook', 'nodes-base.slack'];
+
+    for (const nodeType of criticalNodes) {
+      const node = repository.getNode(nodeType);
+
+      if (!node) {
+        issues.push(`Critical node ${nodeType} not found`);
+        continue;
+      }
+
+      if (node.properties.length === 0) {
+        issues.push(`Node ${nodeType} has no properties`);
+      }
+    }
+
+    // Check AI tools
+    const aiTools = repository.getAITools();
+    if (aiTools.length === 0) {
+      issues.push('No AI tools found - check detection logic');
+    }
+
+    // Check FTS5 table existence and population
+    const ftsTableCheck = db.prepare(`
+      SELECT name FROM sqlite_master
+      WHERE type='table' AND name='nodes_fts'
+    `).get();
+
+    if (!ftsTableCheck) {
+      issues.push('CRITICAL: FTS5 table (nodes_fts) does not exist - searches will fail or be very slow');
+    } else {
+      // Check if FTS5 table is properly populated
+      const ftsCount = db.prepare('SELECT COUNT(*) as count FROM nodes_fts').get() as { count: number };
+
+      if (ftsCount.count === 0) {
+        issues.push('CRITICAL: FTS5 index is empty - searches will return zero results');
+      } else if (nodeCount.count !== ftsCount.count) {
+        issues.push(`FTS5 index out of sync: ${nodeCount.count} nodes but ${ftsCount.count} FTS5 entries`);
+      }
+
+      // Verify critical nodes are searchable via FTS5
+      const searchableNodes = ['webhook', 'merge', 'split'];
+      for (const searchTerm of searchableNodes) {
+        const searchResult = db.prepare(`
+          SELECT COUNT(*) as count FROM nodes_fts
+          WHERE nodes_fts MATCH ?
+        `).get(searchTerm);
+
+        if (searchResult.count === 0) {
+          issues.push(`CRITICAL: Search for "${searchTerm}" returns zero results in FTS5 index`);
+        }
+      }
+    }
+  } catch (error) {
+    // Catch any validation errors
+    const errorMessage = (error as Error).message;
+    issues.push(`Validation error: ${errorMessage}`);
   }
-  
-  // Check AI tools
-  const aiTools = repository.getAITools();
-  if (aiTools.length === 0) {
-    issues.push('No AI tools found - check detection logic');
-  }
-  
+
   return {
     passed: issues.length === 0,
     issues