253b51f5c6
- Fix better-sqlite3 import statements to use namespace import - Update test schemas to match actual database schema - Align NodeRepository tests with actual API implementation - Fix FTS5 tests to work with templates instead of nodes - Update mock data to match ParsedNode interface - Fix column names to match actual schema (node_type, package_name, etc) - Add proper ParsedNode creation helper function - Remove tests for non-existent foreign key constraints
639 lines
20 KiB
TypeScript
639 lines
20 KiB
TypeScript
import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
|
|
import * as Database from 'better-sqlite3';
|
|
import { TestDatabase, TestDataGenerator, PerformanceMonitor } from './test-utils';
|
|
|
|
describe('FTS5 Full-Text Search', () => {
|
|
let testDb: TestDatabase;
|
|
let db: Database;
|
|
|
|
beforeEach(async () => {
|
|
testDb = new TestDatabase({ mode: 'memory', enableFTS5: true });
|
|
db = await testDb.initialize();
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await testDb.cleanup();
|
|
});
|
|
|
|
describe('FTS5 Availability', () => {
|
|
it('should have FTS5 extension available', () => {
|
|
// Try to create an FTS5 table
|
|
expect(() => {
|
|
db.exec('CREATE VIRTUAL TABLE test_fts USING fts5(content)');
|
|
db.exec('DROP TABLE test_fts');
|
|
}).not.toThrow();
|
|
});
|
|
|
|
it('should support FTS5 for template searches', () => {
|
|
// Create FTS5 table for templates
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
// Verify it was created
|
|
const tables = db.prepare(`
|
|
SELECT sql FROM sqlite_master
|
|
WHERE type = 'table' AND name = 'templates_fts'
|
|
`).all() as { sql: string }[];
|
|
|
|
expect(tables).toHaveLength(1);
|
|
expect(tables[0].sql).toContain('USING fts5');
|
|
});
|
|
});
|
|
|
|
describe('Template FTS5 Operations', () => {
|
|
beforeEach(() => {
|
|
// Create FTS5 table
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
// Insert test templates
|
|
const templates = [
|
|
{
|
|
id: 1,
|
|
workflow_id: 1001,
|
|
name: 'Webhook to Slack Notification',
|
|
description: 'Send Slack messages when webhook is triggered',
|
|
nodes_used: JSON.stringify(['n8n-nodes-base.webhook', 'n8n-nodes-base.slack']),
|
|
workflow_json: JSON.stringify({}),
|
|
categories: JSON.stringify([{ id: 1, name: 'automation' }]),
|
|
views: 100
|
|
},
|
|
{
|
|
id: 2,
|
|
workflow_id: 1002,
|
|
name: 'HTTP Request Data Processing',
|
|
description: 'Fetch data from API and process it',
|
|
nodes_used: JSON.stringify(['n8n-nodes-base.httpRequest', 'n8n-nodes-base.set']),
|
|
workflow_json: JSON.stringify({}),
|
|
categories: JSON.stringify([{ id: 2, name: 'data' }]),
|
|
views: 200
|
|
},
|
|
{
|
|
id: 3,
|
|
workflow_id: 1003,
|
|
name: 'Email Automation Workflow',
|
|
description: 'Automate email sending based on triggers',
|
|
nodes_used: JSON.stringify(['n8n-nodes-base.emailSend', 'n8n-nodes-base.if']),
|
|
workflow_json: JSON.stringify({}),
|
|
categories: JSON.stringify([{ id: 3, name: 'communication' }]),
|
|
views: 150
|
|
}
|
|
];
|
|
|
|
const stmt = db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
|
`);
|
|
|
|
templates.forEach(template => {
|
|
stmt.run(
|
|
template.id,
|
|
template.workflow_id,
|
|
template.name,
|
|
template.description,
|
|
template.nodes_used,
|
|
template.workflow_json,
|
|
template.categories,
|
|
template.views
|
|
);
|
|
});
|
|
|
|
// Populate FTS index
|
|
db.exec(`
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
SELECT id, name, description FROM templates
|
|
`);
|
|
});
|
|
|
|
it('should search templates by exact term', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'webhook'
|
|
ORDER BY rank
|
|
`).all();
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({
|
|
name: 'Webhook to Slack Notification'
|
|
});
|
|
});
|
|
|
|
it('should search with partial term and prefix', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'auto*'
|
|
ORDER BY rank
|
|
`).all();
|
|
|
|
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
expect(results.some((r: any) => r.name.includes('Automation'))).toBe(true);
|
|
});
|
|
|
|
it('should search across multiple columns', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'email OR send'
|
|
ORDER BY rank
|
|
`).all();
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({
|
|
name: 'Email Automation Workflow'
|
|
});
|
|
});
|
|
|
|
it('should handle phrase searches', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH '"Slack messages"'
|
|
ORDER BY rank
|
|
`).all();
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({
|
|
name: 'Webhook to Slack Notification'
|
|
});
|
|
});
|
|
|
|
it('should support NOT queries', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'automation NOT email'
|
|
ORDER BY rank
|
|
`).all();
|
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
expect(results.every((r: any) => !r.name.toLowerCase().includes('email'))).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('FTS5 Ranking and Scoring', () => {
|
|
beforeEach(() => {
|
|
// Create FTS5 table
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
// Insert templates with varying relevance
|
|
const templates = [
|
|
{
|
|
id: 1,
|
|
name: 'Advanced HTTP Request Handler',
|
|
description: 'Complex HTTP request processing with error handling and retries'
|
|
},
|
|
{
|
|
id: 2,
|
|
name: 'Simple HTTP GET Request',
|
|
description: 'Basic HTTP GET request example'
|
|
},
|
|
{
|
|
id: 3,
|
|
name: 'Webhook HTTP Receiver',
|
|
description: 'Receive HTTP webhooks and process requests'
|
|
}
|
|
];
|
|
|
|
const stmt = db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, '[]', '{}', '[]', 0, datetime('now'), datetime('now'))
|
|
`);
|
|
|
|
templates.forEach(t => {
|
|
stmt.run(t.id, 1000 + t.id, t.name, t.description);
|
|
});
|
|
|
|
// Populate FTS
|
|
db.exec(`
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
SELECT id, name, description FROM templates
|
|
`);
|
|
});
|
|
|
|
it('should rank results by relevance using bm25', () => {
|
|
const results = db.prepare(`
|
|
SELECT t.*, bm25(templates_fts) as score
|
|
FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'http request'
|
|
ORDER BY bm25(templates_fts)
|
|
`).all() as any[];
|
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
|
|
// Scores should be negative (lower is better in bm25)
|
|
expect(results[0].score).toBeLessThan(0);
|
|
|
|
// Should be ordered by relevance
|
|
expect(results[0].name).toContain('HTTP');
|
|
});
|
|
|
|
it('should use custom weights for columns', () => {
|
|
// Give more weight to name (2.0) than description (1.0)
|
|
const results = db.prepare(`
|
|
SELECT t.*, bm25(templates_fts, 2.0, 1.0) as score
|
|
FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'request'
|
|
ORDER BY bm25(templates_fts, 2.0, 1.0)
|
|
`).all() as any[];
|
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
|
|
// Items with "request" in name should rank higher
|
|
const nameMatches = results.filter((r: any) =>
|
|
r.name.toLowerCase().includes('request')
|
|
);
|
|
expect(nameMatches.length).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
describe('FTS5 Advanced Features', () => {
|
|
beforeEach(() => {
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
// Insert template with longer description
|
|
db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, '[]', '{}', '[]', 0, datetime('now'), datetime('now'))
|
|
`).run(
|
|
1,
|
|
1001,
|
|
'Complex Workflow',
|
|
'This is a complex workflow that handles multiple operations including data transformation, filtering, and aggregation. It can process large datasets efficiently and includes error handling.'
|
|
);
|
|
|
|
db.exec(`
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
SELECT id, name, description FROM templates
|
|
`);
|
|
});
|
|
|
|
it('should support snippet extraction', () => {
|
|
const results = db.prepare(`
|
|
SELECT
|
|
t.*,
|
|
snippet(templates_fts, 1, '<b>', '</b>', '...', 10) as snippet
|
|
FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'transformation'
|
|
`).all() as any[];
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].snippet).toContain('<b>transformation</b>');
|
|
expect(results[0].snippet).toContain('...');
|
|
});
|
|
|
|
it('should support highlight function', () => {
|
|
const results = db.prepare(`
|
|
SELECT
|
|
t.*,
|
|
highlight(templates_fts, 1, '<mark>', '</mark>') as highlighted_desc
|
|
FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'workflow'
|
|
LIMIT 1
|
|
`).all() as any[];
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].highlighted_desc).toContain('<mark>workflow</mark>');
|
|
});
|
|
});
|
|
|
|
describe('FTS5 Triggers and Synchronization', () => {
|
|
beforeEach(() => {
|
|
// Create FTS5 table with triggers
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
);
|
|
|
|
CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates
|
|
BEGIN
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
VALUES (new.id, new.name, new.description);
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates
|
|
BEGIN
|
|
UPDATE templates_fts
|
|
SET name = new.name, description = new.description
|
|
WHERE rowid = new.id;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates
|
|
BEGIN
|
|
DELETE FROM templates_fts WHERE rowid = old.id;
|
|
END;
|
|
`);
|
|
});
|
|
|
|
it('should automatically sync FTS on insert', () => {
|
|
const template = TestDataGenerator.generateTemplate({
|
|
id: 100,
|
|
name: 'Auto-synced Template',
|
|
description: 'This template is automatically indexed'
|
|
});
|
|
|
|
db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
|
`).run(
|
|
template.id,
|
|
template.id + 1000,
|
|
template.name,
|
|
template.description,
|
|
JSON.stringify(template.nodeTypes || []),
|
|
JSON.stringify({}),
|
|
JSON.stringify(template.categories || []),
|
|
template.totalViews || 0
|
|
);
|
|
|
|
// Should immediately be searchable
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'automatically'
|
|
`).all();
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({ id: 100 });
|
|
});
|
|
|
|
it('should automatically sync FTS on update', () => {
|
|
// Insert template
|
|
db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, '[]', '{}', '[]', 0, datetime('now'), datetime('now'))
|
|
`).run(200, 2000, 'Original Name', 'Original description');
|
|
|
|
// Update description
|
|
db.prepare(`
|
|
UPDATE templates
|
|
SET description = 'Updated description with new keywords'
|
|
WHERE id = ?
|
|
`).run(200);
|
|
|
|
// Should find with new keywords
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'keywords'
|
|
`).all();
|
|
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({ id: 200 });
|
|
|
|
// Should not find with old keywords
|
|
const oldResults = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH 'Original'
|
|
`).all();
|
|
|
|
expect(oldResults).toHaveLength(0);
|
|
});
|
|
|
|
it('should automatically sync FTS on delete', () => {
|
|
// Insert template
|
|
db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, '[]', '{}', '[]', 0, datetime('now'), datetime('now'))
|
|
`).run(300, 3000, 'Temporary Template', 'This will be deleted');
|
|
|
|
// Verify it's searchable
|
|
let count = db.prepare(`
|
|
SELECT COUNT(*) as count
|
|
FROM templates_fts
|
|
WHERE templates_fts MATCH 'Temporary'
|
|
`).get() as { count: number };
|
|
expect(count.count).toBe(1);
|
|
|
|
// Delete template
|
|
db.prepare('DELETE FROM templates WHERE id = ?').run(300);
|
|
|
|
// Should no longer be searchable
|
|
count = db.prepare(`
|
|
SELECT COUNT(*) as count
|
|
FROM templates_fts
|
|
WHERE templates_fts MATCH 'Temporary'
|
|
`).get() as { count: number };
|
|
expect(count.count).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('FTS5 Performance', () => {
|
|
it('should handle large dataset searches efficiently', () => {
|
|
// Create FTS5 table
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
const monitor = new PerformanceMonitor();
|
|
|
|
// Insert a large number of templates
|
|
const templates = TestDataGenerator.generateTemplates(1000);
|
|
const insertStmt = db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
|
`);
|
|
|
|
const insertMany = db.transaction((templates: any[]) => {
|
|
templates.forEach((template, i) => {
|
|
insertStmt.run(
|
|
i + 1,
|
|
template.id,
|
|
template.name,
|
|
template.description || `Template ${i} for ${['webhook handling', 'API calls', 'data processing', 'automation'][i % 4]}`,
|
|
JSON.stringify(template.nodeTypes || []),
|
|
JSON.stringify(template.workflowInfo || {}),
|
|
JSON.stringify(template.categories || []),
|
|
template.totalViews || 0
|
|
);
|
|
});
|
|
|
|
// Populate FTS in bulk
|
|
db.exec(`
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
SELECT id, name, description FROM templates
|
|
`);
|
|
});
|
|
|
|
const stopInsert = monitor.start('bulk_insert');
|
|
insertMany(templates);
|
|
stopInsert();
|
|
|
|
// Test search performance
|
|
const searchTerms = ['workflow', 'webhook', 'automation', 'data processing', 'api'];
|
|
|
|
searchTerms.forEach(term => {
|
|
const stop = monitor.start(`search_${term}`);
|
|
const results = db.prepare(`
|
|
SELECT t.* FROM templates t
|
|
JOIN templates_fts f ON t.id = f.rowid
|
|
WHERE templates_fts MATCH ?
|
|
ORDER BY rank
|
|
LIMIT 10
|
|
`).all(term);
|
|
stop();
|
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
// All searches should complete quickly
|
|
searchTerms.forEach(term => {
|
|
const stats = monitor.getStats(`search_${term}`);
|
|
expect(stats).not.toBeNull();
|
|
expect(stats!.average).toBeLessThan(10); // Should complete in under 10ms
|
|
});
|
|
});
|
|
|
|
it('should optimize rebuilding FTS index', () => {
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
|
|
// Insert initial data
|
|
const templates = TestDataGenerator.generateTemplates(100);
|
|
const insertStmt = db.prepare(`
|
|
INSERT INTO templates (
|
|
id, workflow_id, name, description,
|
|
nodes_used, workflow_json, categories, views,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, '[]', '{}', '[]', 0, datetime('now'), datetime('now'))
|
|
`);
|
|
|
|
db.transaction(() => {
|
|
templates.forEach((template, i) => {
|
|
insertStmt.run(
|
|
i + 1,
|
|
template.id,
|
|
template.name,
|
|
template.description || 'Test template'
|
|
);
|
|
});
|
|
|
|
db.exec(`
|
|
INSERT INTO templates_fts(rowid, name, description)
|
|
SELECT id, name, description FROM templates
|
|
`);
|
|
})();
|
|
|
|
// Rebuild FTS index
|
|
const monitor = new PerformanceMonitor();
|
|
const stop = monitor.start('rebuild_fts');
|
|
|
|
db.exec('INSERT INTO templates_fts(templates_fts) VALUES("rebuild")');
|
|
|
|
stop();
|
|
|
|
const stats = monitor.getStats('rebuild_fts');
|
|
expect(stats).not.toBeNull();
|
|
expect(stats!.average).toBeLessThan(100); // Should complete quickly
|
|
});
|
|
});
|
|
|
|
describe('FTS5 Error Handling', () => {
|
|
beforeEach(() => {
|
|
db.exec(`
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
|
name,
|
|
description,
|
|
content=templates,
|
|
content_rowid=id
|
|
)
|
|
`);
|
|
});
|
|
|
|
it('should handle malformed queries gracefully', () => {
|
|
expect(() => {
|
|
db.prepare(`
|
|
SELECT * FROM templates_fts WHERE templates_fts MATCH ?
|
|
`).all('AND OR NOT'); // Invalid query syntax
|
|
}).toThrow(/fts5: syntax error/);
|
|
});
|
|
|
|
it('should handle special characters in search terms', () => {
|
|
const specialChars = ['@', '#', '$', '%', '^', '&', '*', '(', ')'];
|
|
|
|
specialChars.forEach(char => {
|
|
// Should not throw when properly escaped
|
|
const results = db.prepare(`
|
|
SELECT * FROM templates_fts WHERE templates_fts MATCH ?
|
|
`).all(`"${char}"`);
|
|
|
|
expect(Array.isArray(results)).toBe(true);
|
|
});
|
|
});
|
|
|
|
it('should handle empty search terms', () => {
|
|
const results = db.prepare(`
|
|
SELECT * FROM templates_fts WHERE templates_fts MATCH ?
|
|
`).all('');
|
|
|
|
expect(results).toHaveLength(0);
|
|
});
|
|
});
|
|
}); |