ros/n8n-mcp
1
0
Fork 0
mirror of https://github.com/czlonkowski/n8n-mcp.git synced 2026-02-09 06:43:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add template metadata generation and smart discovery

Browse Source 
- Implement OpenAI batch API integration for metadata generation
- Add search_templates_by_metadata tool with advanced filtering
- Enhance list_templates to include descriptions and optional metadata
- Generate metadata for 2,534 templates (97.5% coverage)
- Update README with Template Tools section and enhanced Claude setup
- Add comprehensive documentation for metadata system

Enables intelligent template discovery through:
- Complexity levels (simple/medium/complex)
- Setup time estimates (5-480 minutes)
- Target audience filtering (developers/marketers/analysts)
- Required services detection
- Category and use case classification

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
2025-09-15 00:18:53 +02:00
parent 6e24da722b
commit 1e586c0b23
15 changed files with 1159 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/mcp/server.ts
View File

@@ -730,7 +730,8 @@ export class N8NDocumentationMCPServer {
const listLimit = Math.min(Math.max(Number(args.limit) || 10, 1), 100);
const listOffset = Math.max(Number(args.offset) || 0, 0);
const sortBy = args.sortBy || 'views';
return this.listTemplates(listLimit, listOffset, sortBy);
const includeMetadata = Boolean(args.includeMetadata);
return this.listTemplates(listLimit, listOffset, sortBy, includeMetadata);
case 'list_node_templates':
this.validateToolParams(name, args, ['nodeTypes']);
const templateLimit = Math.min(Math.max(Number(args.limit) || 10, 1), 100);
@@ -751,6 +752,18 @@ export class N8NDocumentationMCPServer {
const taskLimit = Math.min(Math.max(Number(args.limit) || 10, 1), 100);
const taskOffset = Math.max(Number(args.offset) || 0, 0);
return this.getTemplatesForTask(args.task, taskLimit, taskOffset);
case 'search_templates_by_metadata':
// No required params - all filters are optional
const metadataLimit = Math.min(Math.max(Number(args.limit) || 20, 1), 100);
const metadataOffset = Math.max(Number(args.offset) || 0, 0);
return this.searchTemplatesByMetadata({
category: args.category,
complexity: args.complexity,
maxSetupMinutes: args.maxSetupMinutes ? Number(args.maxSetupMinutes) : undefined,
minSetupMinutes: args.minSetupMinutes ? Number(args.minSetupMinutes) : undefined,
requiredService: args.requiredService,
targetAudience: args.targetAudience
}, metadataLimit, metadataOffset);
case 'validate_workflow':
this.validateToolParams(name, args, ['workflow']);
return this.validateWorkflow(args.workflow, args.options);
@@ -2328,11 +2341,11 @@ Full documentation is being prepared. For now, use get_node_essentials for confi
}
// Template-related methods
private async listTemplates(limit: number = 10, offset: number = 0, sortBy: 'views' | 'created_at' | 'name' = 'views'): Promise<any> {
private async listTemplates(limit: number = 10, offset: number = 0, sortBy: 'views' | 'created_at' | 'name' = 'views', includeMetadata: boolean = false): Promise<any> {
await this.ensureInitialized();
if (!this.templateService) throw new Error('Template service not initialized');
const result = await this.templateService.listTemplates(limit, offset, sortBy);
const result = await this.templateService.listTemplates(limit, offset, sortBy, includeMetadata);
return {
...result,
@@ -2431,6 +2444,50 @@ Full documentation is being prepared. For now, use get_node_essentials for confi
};
}
private async searchTemplatesByMetadata(filters: {
category?: string;
complexity?: 'simple' | 'medium' | 'complex';
maxSetupMinutes?: number;
minSetupMinutes?: number;
requiredService?: string;
targetAudience?: string;
}, limit: number = 20, offset: number = 0): Promise<any> {
await this.ensureInitialized();
if (!this.templateService) throw new Error('Template service not initialized');
const result = await this.templateService.searchTemplatesByMetadata(filters, limit, offset);
// Build filter summary for feedback
const filterSummary: string[] = [];
if (filters.category) filterSummary.push(`category: ${filters.category}`);
if (filters.complexity) filterSummary.push(`complexity: ${filters.complexity}`);
if (filters.maxSetupMinutes) filterSummary.push(`max setup: ${filters.maxSetupMinutes} min`);
if (filters.minSetupMinutes) filterSummary.push(`min setup: ${filters.minSetupMinutes} min`);
if (filters.requiredService) filterSummary.push(`service: ${filters.requiredService}`);
if (filters.targetAudience) filterSummary.push(`audience: ${filters.targetAudience}`);
if (result.items.length === 0 && offset === 0) {
// Get available categories and audiences for suggestions
const availableCategories = await this.templateService.getAvailableCategories();
const availableAudiences = await this.templateService.getAvailableTargetAudiences();
return {
...result,
message: `No templates found with filters: ${filterSummary.join(', ')}`,
availableCategories: availableCategories.slice(0, 10),
availableAudiences: availableAudiences.slice(0, 5),
tip: "Try broader filters or different categories. Use list_templates to see all templates."
};
}
return {
...result,
filters,
filterSummary: filterSummary.join(', '),
tip: `Found ${result.total} templates matching filters. Showing ${result.items.length}. Each includes AI-generated metadata.`
};
}
private getTaskDescription(task: string): string {
const descriptions: Record<string, string> = {
'ai_automation': 'AI-powered workflows using OpenAI, LangChain, and other AI tools',

4
src/mcp/tool-docs/index.ts
View File

@@ -22,7 +22,8 @@ import {
getNodeForTaskDoc,
listNodeTemplatesDoc,
getTemplateDoc,
searchTemplatesDoc,
searchTemplatesDoc,
searchTemplatesByMetadataDoc,
getTemplatesForTaskDoc
} from './templates';
import {
@@ -83,6 +84,7 @@ export const toolsDocumentation: Record<string, ToolDocumentation> = {
list_node_templates: listNodeTemplatesDoc,
get_template: getTemplateDoc,
search_templates: searchTemplatesDoc,
search_templates_by_metadata: searchTemplatesByMetadataDoc,
get_templates_for_task: getTemplatesForTaskDoc,
// Workflow Management tools (n8n API)

1
src/mcp/tool-docs/templates/index.ts
View File

@@ -3,4 +3,5 @@ export { listTasksDoc } from './list-tasks';
export { listNodeTemplatesDoc } from './list-node-templates';
export { getTemplateDoc } from './get-template';
export { searchTemplatesDoc } from './search-templates';
export { searchTemplatesByMetadataDoc } from './search-templates-by-metadata';
export { getTemplatesForTaskDoc } from './get-templates-for-task';

118
src/mcp/tool-docs/templates/search-templates-by-metadata.ts Normal file
View File

@@ -0,0 +1,118 @@
import { ToolDocumentation } from '../types';
export const searchTemplatesByMetadataDoc: ToolDocumentation = {
name: 'search_templates_by_metadata',
category: 'templates',
essentials: {
description: 'Search templates using AI-generated metadata filters. Find templates by complexity, setup time, required services, or target audience. Enables smart template discovery beyond simple text search.',
keyParameters: ['category', 'complexity', 'maxSetupMinutes', 'targetAudience'],
example: 'search_templates_by_metadata({complexity: "simple", maxSetupMinutes: 30})',
performance: 'Fast (<100ms) - JSON extraction queries',
tips: [
'All filters are optional - combine them for precise results',
'Use getAvailableCategories() to see valid category values',
'Complexity levels: simple, medium, complex',
'Setup time is in minutes (5-480 range)'
]
},
full: {
description: `Advanced template search using AI-generated metadata. Each template has been analyzed by GPT-4 to extract structured information about its purpose, complexity, setup requirements, and target users. This enables intelligent filtering beyond simple keyword matching, helping you find templates that match your specific needs, skill level, and available time.`,
parameters: {
category: {
type: 'string',
required: false,
description: 'Filter by category like "automation", "integration", "data processing", "communication". Use template service getAvailableCategories() for full list.'
},
complexity: {
type: 'string (enum)',
required: false,
description: 'Filter by implementation complexity: "simple" (beginner-friendly), "medium" (some experience needed), or "complex" (advanced features)'
},
maxSetupMinutes: {
type: 'number',
required: false,
description: 'Maximum acceptable setup time in minutes (5-480). Find templates you can implement within your time budget.'
},
minSetupMinutes: {
type: 'number',
required: false,
description: 'Minimum setup time in minutes (5-480). Find more substantial templates that offer comprehensive solutions.'
},
requiredService: {
type: 'string',
required: false,
description: 'Filter by required external service like "openai", "slack", "google", "shopify". Ensures you have necessary accounts/APIs.'
},
targetAudience: {
type: 'string',
required: false,
description: 'Filter by intended users: "developers", "marketers", "analysts", "operations", "sales". Find templates for your role.'
},
limit: {
type: 'number',
required: false,
description: 'Maximum results to return. Default 20, max 100.'
},
offset: {
type: 'number',
required: false,
description: 'Pagination offset for results. Default 0.'
}
},
returns: `Returns an object containing:
- items: Array of matching templates with full metadata
- id: Template ID
- name: Template name
- description: Purpose and functionality
- author: Creator details
- nodes: Array of nodes used
- views: Popularity count
- metadata: AI-generated structured data
- categories: Primary use categories
- complexity: Difficulty level
- use_cases: Specific applications
- estimated_setup_minutes: Time to implement
- required_services: External dependencies
- key_features: Main capabilities
- target_audience: Intended users
- total: Total matching templates
- filters: Applied filter criteria
- filterSummary: Human-readable filter description
- availableCategories: Suggested categories if no results
- availableAudiences: Suggested audiences if no results
- tip: Contextual guidance`,
examples: [
'search_templates_by_metadata({complexity: "simple"}) - Find beginner-friendly templates',
'search_templates_by_metadata({category: "automation", maxSetupMinutes: 30}) - Quick automation templates',
'search_templates_by_metadata({targetAudience: "marketers"}) - Marketing-focused workflows',
'search_templates_by_metadata({requiredService: "openai", complexity: "medium"}) - AI templates with moderate complexity',
'search_templates_by_metadata({minSetupMinutes: 60, category: "integration"}) - Comprehensive integration solutions'
],
useCases: [
'Finding beginner-friendly templates by setting complexity:"simple"',
'Discovering templates you can implement quickly with maxSetupMinutes:30',
'Finding role-specific workflows with targetAudience filter',
'Identifying templates that need specific APIs with requiredService filter',
'Combining multiple filters for precise template discovery'
],
performance: 'Fast (<100ms) - Uses SQLite JSON extraction on pre-generated metadata. 97.5% coverage (2,534/2,598 templates).',
bestPractices: [
'Start with broad filters and narrow down based on results',
'Use getAvailableCategories() to discover valid category values',
'Combine complexity and setup time for skill-appropriate templates',
'Check required services before selecting templates to ensure you have necessary accounts'
],
pitfalls: [
'Not all templates have metadata (97.5% coverage)',
'Setup time estimates assume basic n8n familiarity',
'Categories/audiences use partial matching - be specific',
'Metadata is AI-generated and may occasionally be imprecise'
],
relatedTools: [
'list_templates',
'search_templates',
'list_node_templates',
'get_templates_for_task'
]
}
};

58
src/mcp/tools.ts
View File

@@ -325,7 +325,7 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
},
{
name: 'list_templates',
description: `List all templates with minimal data (id, name, views, node count). Use for browsing available templates.`,
description: `List all templates with minimal data (id, name, description, views, node count). Optionally include AI-generated metadata for smart filtering.`,
inputSchema: {
type: 'object',
properties: {
@@ -348,6 +348,11 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
description: 'Sort field. Default: views (popularity).',
default: 'views',
},
includeMetadata: {
type: 'boolean',
description: 'Include AI-generated metadata (categories, complexity, setup time, etc.). Default false.',
default: false,
},
},
},
},
@@ -465,6 +470,57 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
required: ['task'],
},
},
{
name: 'search_templates_by_metadata',
description: `Search templates by AI-generated metadata. Filter by category, complexity, setup time, services, or audience. Returns rich metadata for smart template discovery.`,
inputSchema: {
type: 'object',
properties: {
category: {
type: 'string',
description: 'Filter by category (e.g., "automation", "integration", "data processing")',
},
complexity: {
type: 'string',
enum: ['simple', 'medium', 'complex'],
description: 'Filter by complexity level',
},
maxSetupMinutes: {
type: 'number',
description: 'Maximum setup time in minutes',
minimum: 5,
maximum: 480,
},
minSetupMinutes: {
type: 'number',
description: 'Minimum setup time in minutes',
minimum: 5,
maximum: 480,
},
requiredService: {
type: 'string',
description: 'Filter by required service (e.g., "openai", "slack", "google")',
},
targetAudience: {
type: 'string',
description: 'Filter by target audience (e.g., "developers", "marketers", "analysts")',
},
limit: {
type: 'number',
description: 'Maximum number of results. Default 20.',
default: 20,
minimum: 1,
maximum: 100,
},
offset: {
type: 'number',
description: 'Pagination offset. Default 0.',
default: 0,
minimum: 0,
},
},
},
},
{
name: 'validate_workflow',
description: `Full workflow validation: structure, connections, expressions, AI tools. Returns errors/warnings/fixes. Essential before deploy.`,

164
src/scripts/fetch-templates.ts
View File

@@ -3,13 +3,34 @@ import { createDatabaseAdapter } from '../database/database-adapter';
import { TemplateService } from '../templates/template-service';
import * as fs from 'fs';
import * as path from 'path';
import * as zlib from 'zlib';
import * as dotenv from 'dotenv';
import type { MetadataRequest } from '../templates/metadata-generator';
// Load environment variables
dotenv.config();
async function fetchTemplates(mode: 'rebuild' | 'update' = 'rebuild', generateMetadata: boolean = false) {
async function fetchTemplates(mode: 'rebuild' | 'update' = 'rebuild', generateMetadata: boolean = false, metadataOnly: boolean = false) {
// If metadata-only mode, skip template fetching entirely
if (metadataOnly) {
console.log('🤖 Metadata-only mode: Generating metadata for existing templates...\n');
if (!process.env.OPENAI_API_KEY) {
console.error('❌ OPENAI_API_KEY not set in environment');
process.exit(1);
}
const db = await createDatabaseAdapter('./data/nodes.db');
const service = new TemplateService(db);
await generateTemplateMetadata(db, service);
if ('close' in db && typeof db.close === 'function') {
db.close();
}
return;
}
const modeEmoji = mode === 'rebuild' ? '🔄' : '⬆️';
const modeText = mode === 'rebuild' ? 'Rebuilding' : 'Updating';
console.log(`${modeEmoji} ${modeText} n8n workflow templates...\n`);
@@ -27,66 +48,48 @@ async function fetchTemplates(mode: 'rebuild' | 'update' = 'rebuild', generateMe
// Initialize database
const db = await createDatabaseAdapter('./data/nodes.db');
// Only drop tables in rebuild mode
// Handle database schema based on mode
if (mode === 'rebuild') {
try {
// Drop existing tables in rebuild mode
db.exec('DROP TABLE IF EXISTS templates');
db.exec('DROP TABLE IF EXISTS templates_fts');
console.log('🗑️ Dropped existing templates tables (rebuild mode)\n');
// Apply fresh schema
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
db.exec(schema);
console.log('📋 Applied database schema\n');
} catch (error) {
// Ignore errors if tables don't exist
console.error('❌ Error setting up database schema:', error);
throw error;
}
} else {
console.log('📊 Update mode: Keeping existing templates\n');
}
// Apply schema with updated constraint
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
db.exec(schema);
// Pre-create FTS5 tables if supported
const hasFTS5 = db.checkFTS5Support();
if (hasFTS5) {
console.log('🔍 Creating FTS5 tables for template search...');
console.log('📊 Update mode: Keeping existing templates and schema\n');
// In update mode, only ensure new columns exist (for migration)
try {
// Create FTS5 virtual table
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
name, description, content=templates
);
`);
// Check if metadata columns exist, add them if not (migration support)
const columns = db.prepare("PRAGMA table_info(templates)").all() as any[];
const hasMetadataColumn = columns.some((col: any) => col.name === 'metadata_json');
// Create triggers to keep FTS5 in sync
db.exec(`
CREATE TRIGGER IF NOT EXISTS templates_ai AFTER INSERT ON templates BEGIN
INSERT INTO templates_fts(rowid, name, description)
VALUES (new.id, new.name, new.description);
END;
`);
db.exec(`
CREATE TRIGGER IF NOT EXISTS templates_au AFTER UPDATE ON templates BEGIN
UPDATE templates_fts SET name = new.name, description = new.description
WHERE rowid = new.id;
END;
`);
db.exec(`
CREATE TRIGGER IF NOT EXISTS templates_ad AFTER DELETE ON templates BEGIN
DELETE FROM templates_fts WHERE rowid = old.id;
END;
`);
console.log('✅ FTS5 tables created successfully\n');
if (!hasMetadataColumn) {
console.log('📋 Adding metadata columns to existing schema...');
db.exec(`
ALTER TABLE templates ADD COLUMN metadata_json TEXT;
ALTER TABLE templates ADD COLUMN metadata_generated_at DATETIME;
`);
console.log('✅ Metadata columns added\n');
}
} catch (error) {
console.log('⚠️ Failed to create FTS5 tables:', error);
console.log(' Template search will use LIKE fallback\n');
// Columns might already exist, that's fine
console.log('📋 Schema is up to date\n');
}
} else {
console.log('ℹ️ FTS5 not supported in this SQLite build');
console.log(' Template search will use LIKE queries\n');
}
// FTS5 initialization is handled by TemplateRepository
// No need to duplicate the logic here
// Create service
const service = new TemplateService(db);
@@ -104,7 +107,7 @@ async function fetchTemplates(mode: 'rebuild' | 'update' = 'rebuild', generateMe
const progress = total > 0 ? Math.round((current / total) * 100) : 0;
lastMessage = `📊 ${message}: ${current}/${total} (${progress}%)`;
process.stdout.write(lastMessage);
}, mode);
}, mode); // Pass the mode parameter!
console.log('\n'); // New line after progress
@@ -148,8 +151,11 @@ async function generateTemplateMetadata(db: any, service: TemplateService) {
const { BatchProcessor } = await import('../templates/batch-processor');
const repository = (service as any).repository;
// Get templates without metadata
const templatesWithoutMetadata = repository.getTemplatesWithoutMetadata(500);
// Get templates without metadata (0 = no limit)
const limit = parseInt(process.env.METADATA_LIMIT || '0');
const templatesWithoutMetadata = limit > 0
? repository.getTemplatesWithoutMetadata(limit)
: repository.getTemplatesWithoutMetadata(999999); // Get all
if (templatesWithoutMetadata.length === 0) {
console.log('✅ All templates already have metadata');
@@ -159,23 +165,44 @@ async function generateTemplateMetadata(db: any, service: TemplateService) {
console.log(`Found ${templatesWithoutMetadata.length} templates without metadata`);
// Create batch processor
const batchSize = parseInt(process.env.OPENAI_BATCH_SIZE || '50');
console.log(`Processing in batches of ${batchSize} templates each`);
// Warn if batch size is very large
if (batchSize > 100) {
console.log(`⚠️ Large batch size (${batchSize}) may take longer to process`);
console.log(` Consider using OPENAI_BATCH_SIZE=50 for faster results`);
}
const processor = new BatchProcessor({
apiKey: process.env.OPENAI_API_KEY!,
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
batchSize: parseInt(process.env.OPENAI_BATCH_SIZE || '100'),
batchSize: batchSize,
outputDir: './temp/batch'
});
// Prepare metadata requests
const requests: MetadataRequest[] = templatesWithoutMetadata.map((t: any) => ({
templateId: t.id,
name: t.name,
description: t.description,
nodes: JSON.parse(t.nodes_used),
workflow: t.workflow_json_compressed
? JSON.parse(Buffer.from(t.workflow_json_compressed, 'base64').toString())
: (t.workflow_json ? JSON.parse(t.workflow_json) : undefined)
}));
const requests: MetadataRequest[] = templatesWithoutMetadata.map((t: any) => {
let workflow = undefined;
try {
if (t.workflow_json_compressed) {
const decompressed = zlib.gunzipSync(Buffer.from(t.workflow_json_compressed, 'base64'));
workflow = JSON.parse(decompressed.toString());
} else if (t.workflow_json) {
workflow = JSON.parse(t.workflow_json);
}
} catch (error) {
console.warn(`Failed to parse workflow for template ${t.id}:`, error);
}
return {
templateId: t.id,
name: t.name,
description: t.description,
nodes: JSON.parse(t.nodes_used),
workflow
};
});
// Process in batches
const results = await processor.processTemplates(requests, (message, current, total) => {
@@ -210,11 +237,12 @@ async function generateTemplateMetadata(db: any, service: TemplateService) {
}
// Parse command line arguments
function parseArgs(): { mode: 'rebuild' | 'update', generateMetadata: boolean } {
function parseArgs(): { mode: 'rebuild' | 'update', generateMetadata: boolean, metadataOnly: boolean } {
const args = process.argv.slice(2);
let mode: 'rebuild' | 'update' = 'rebuild';
let generateMetadata = false;
let metadataOnly = false;
// Check for --mode flag
const modeIndex = args.findIndex(arg => arg.startsWith('--mode'));
@@ -237,25 +265,31 @@ function parseArgs(): { mode: 'rebuild' | 'update', generateMetadata: boolean }
generateMetadata = true;
}
// Check for --metadata-only flag
if (args.includes('--metadata-only')) {
metadataOnly = true;
}
// Show help if requested
if (args.includes('--help') || args.includes('-h')) {
console.log('Usage: npm run fetch:templates [options]\n');
console.log('Options:');
console.log(' --mode=rebuild|update Rebuild from scratch or update existing (default: rebuild)');
console.log(' --update Shorthand for --mode=update');
console.log(' --generate-metadata Generate AI metadata for templates (requires OPENAI_API_KEY)');
console.log(' --generate-metadata Generate AI metadata after fetching templates');
console.log(' --metadata Shorthand for --generate-metadata');
console.log(' --metadata-only Only generate metadata, skip template fetching');
console.log(' --help, -h Show this help message');
process.exit(0);
}
return { mode, generateMetadata };
return { mode, generateMetadata, metadataOnly };
}
// Run if called directly
if (require.main === module) {
const { mode, generateMetadata } = parseArgs();
fetchTemplates(mode, generateMetadata).catch(console.error);
const { mode, generateMetadata, metadataOnly } = parseArgs();
fetchTemplates(mode, generateMetadata, metadataOnly).catch(console.error);
}
export { fetchTemplates };

121
src/templates/batch-processor.ts
View File

@@ -40,7 +40,7 @@ export class BatchProcessor {
}
/**
* Process templates in batches
* Process templates in batches (parallel submission)
*/
async processTemplates(
templates: MetadataRequest[],
@@ -51,26 +51,62 @@ export class BatchProcessor {
logger.info(`Processing ${templates.length} templates in ${batches.length} batches`);
// Submit all batches in parallel
console.log(`\n📤 Submitting ${batches.length} batch${batches.length > 1 ? 'es' : ''} to OpenAI...`);
const batchJobs: Array<{ batchNum: number; jobPromise: Promise<any>; templates: MetadataRequest[] }> = [];
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
const batchNum = i + 1;
try {
progressCallback?.(`Processing batch ${batchNum}/${batches.length}`, i * this.batchSize, templates.length);
progressCallback?.(`Submitting batch ${batchNum}/${batches.length}`, i * this.batchSize, templates.length);
// Process this batch
const batchResults = await this.processBatch(batch, `batch_${batchNum}`);
// Submit batch (don't wait for completion)
const jobPromise = this.submitBatch(batch, `batch_${batchNum}`);
batchJobs.push({ batchNum, jobPromise, templates: batch });
// Merge results
for (const result of batchResults) {
results.set(result.templateId, result);
}
console.log(` 📨 Submitted batch ${batchNum}/${batches.length} (${batch.length} templates)`);
} catch (error) {
logger.error(`Error submitting batch ${batchNum}:`, error);
console.error(` ❌ Failed to submit batch ${batchNum}`);
}
}
console.log(`\n⏳ All batches submitted. Waiting for completion...`);
console.log(` (Batches process in parallel - this is much faster than sequential processing)`);
// Process all batches in parallel and collect results as they complete
const batchPromises = batchJobs.map(async ({ batchNum, jobPromise, templates: batchTemplates }) => {
try {
const completedJob = await jobPromise;
console.log(`\n📦 Retrieving results for batch ${batchNum}/${batches.length}...`);
logger.info(`Completed batch ${batchNum}/${batches.length}: ${batchResults.length} results`);
progressCallback?.(`Completed batch ${batchNum}/${batches.length}`, Math.min((i + 1) * this.batchSize, templates.length), templates.length);
// Retrieve and parse results
const batchResults = await this.retrieveResults(completedJob);
logger.info(`Retrieved ${batchResults.length} results from batch ${batchNum}`);
progressCallback?.(`Retrieved batch ${batchNum}/${batches.length}`,
Math.min(batchNum * this.batchSize, templates.length), templates.length);
return { batchNum, results: batchResults };
} catch (error) {
logger.error(`Error processing batch ${batchNum}:`, error);
// Continue with next batch
console.error(` ❌ Batch ${batchNum} failed:`, error);
return { batchNum, results: [] };
}
});
// Wait for all batches to complete
const allBatchResults = await Promise.all(batchPromises);
// Merge all results
for (const { batchNum, results: batchResults } of allBatchResults) {
for (const result of batchResults) {
results.set(result.templateId, result);
}
if (batchResults.length > 0) {
console.log(` ✅ Merged ${batchResults.length} results from batch ${batchNum}`);
}
}
@@ -78,6 +114,51 @@ export class BatchProcessor {
return results;
}
/**
* Submit a batch without waiting for completion
*/
private async submitBatch(templates: MetadataRequest[], batchName: string): Promise<any> {
// Create JSONL file
const inputFile = await this.createBatchFile(templates, batchName);
try {
// Upload file to OpenAI
const uploadedFile = await this.uploadFile(inputFile);
// Create batch job
const batchJob = await this.createBatchJob(uploadedFile.id);
// Start monitoring (returns promise that resolves when complete)
const monitoringPromise = this.monitorBatchJob(batchJob.id);
// Clean up input file immediately
try {
fs.unlinkSync(inputFile);
} catch {}
// Store file IDs for cleanup later
monitoringPromise.then(async (completedJob) => {
// Cleanup uploaded files after completion
try {
await this.client.files.del(uploadedFile.id);
if (completedJob.output_file_id) {
// Note: We'll delete output file after retrieving results
}
} catch (error) {
logger.warn(`Failed to cleanup files for batch ${batchName}`, error);
}
});
return monitoringPromise;
} catch (error) {
// Cleanup on error
try {
fs.unlinkSync(inputFile);
} catch {}
throw error;
}
}
/**
* Process a single batch
*/
@@ -180,17 +261,33 @@ export class BatchProcessor {
* Monitor batch job with exponential backoff
*/
private async monitorBatchJob(batchId: string): Promise<any> {
const waitTimes = [60, 120, 300, 600, 900, 1800]; // Progressive wait times in seconds
// Start with shorter wait times for better UX
const waitTimes = [30, 60, 120, 300, 600, 900, 1800]; // Progressive wait times in seconds
let waitIndex = 0;
let attempts = 0;
const maxAttempts = 100; // Safety limit
const startTime = Date.now();
let lastStatus = '';
while (attempts < maxAttempts) {
const batchJob = await this.client.batches.retrieve(batchId);
// Only log if status changed
if (batchJob.status !== lastStatus) {
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
const statusSymbol = batchJob.status === 'in_progress' ? '⚙️' :
batchJob.status === 'finalizing' ? '📦' :
batchJob.status === 'validating' ? '🔍' : '⏳';
console.log(` ${statusSymbol} Batch ${batchId.slice(-8)}: ${batchJob.status} (${elapsedMinutes} min)`);
lastStatus = batchJob.status;
}
logger.debug(`Batch ${batchId} status: ${batchJob.status} (attempt ${attempts + 1})`);
if (batchJob.status === 'completed') {
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
console.log(` ✅ Batch ${batchId.slice(-8)} completed in ${elapsedMinutes} minutes`);
logger.info(`Batch job ${batchId} completed successfully`);
return batchJob;
}

26
src/templates/metadata-generator.ts
View File

@@ -125,8 +125,8 @@ export class MetadataGenerator {
url: '/v1/chat/completions',
body: {
model: this.model,
temperature: 0.1,
max_tokens: 500,
temperature: 1,
max_completion_tokens: 1000,
response_format: {
type: 'json_schema',
json_schema: this.getJsonSchema()
@@ -134,18 +134,7 @@ export class MetadataGenerator {
messages: [
{
role: 'system',
content: `You are an n8n workflow expert analyzing templates to extract structured metadata.
Analyze the provided template information and extract:
- Categories: Classify into relevant categories (automation, integration, data, communication, etc.)
- Complexity: Assess as simple (1-3 nodes), medium (4-8 nodes), or complex (9+ nodes or advanced logic)
- Use cases: Identify primary business use cases
- Setup time: Estimate realistic setup time based on complexity and required configurations
- Required services: List any external services, APIs, or accounts needed
- Key features: Highlight main capabilities or benefits
- Target audience: Identify who would benefit most (developers, marketers, ops teams, etc.)
Be concise and practical in your analysis.`
content: `Analyze n8n workflow templates and extract metadata. Be concise.`
},
{
role: 'user',
@@ -254,8 +243,8 @@ export class MetadataGenerator {
try {
const completion = await this.client.chat.completions.create({
model: this.model,
temperature: 0.1,
max_tokens: 500,
temperature: 1,
max_completion_tokens: 1000,
response_format: {
type: 'json_schema',
json_schema: this.getJsonSchema()
@@ -263,17 +252,18 @@ export class MetadataGenerator {
messages: [
{
role: 'system',
content: `You are an n8n workflow expert analyzing templates to extract structured metadata.`
content: `Analyze n8n workflow templates and extract metadata. Be concise.`
},
{
role: 'user',
content: `Analyze this template: ${template.name}\nNodes: ${template.nodes.join(', ')}`
content: `Template: ${template.name}\nNodes: ${template.nodes.slice(0, 10).join(', ')}`
}
]
});
const content = completion.choices[0].message.content;
if (!content) {
logger.error('No content in OpenAI response');
throw new Error('No content in response');
}

169
src/templates/template-repository.ts
View File

@@ -625,4 +625,173 @@ export class TemplateRepository {
return { total, withMetadata, withoutMetadata, outdated };
}
/**
* Search templates by metadata fields
*/
searchTemplatesByMetadata(filters: {
category?: string;
complexity?: 'simple' | 'medium' | 'complex';
maxSetupMinutes?: number;
minSetupMinutes?: number;
requiredService?: string;
targetAudience?: string;
}, limit: number = 20, offset: number = 0): StoredTemplate[] {
const conditions: string[] = ['metadata_json IS NOT NULL'];
const params: any[] = [];
// Build WHERE conditions based on filters
if (filters.category) {
conditions.push("json_extract(metadata_json, '$.categories') LIKE ?");
params.push(`%"${filters.category}"%`);
}
if (filters.complexity) {
conditions.push("json_extract(metadata_json, '$.complexity') = ?");
params.push(filters.complexity);
}
if (filters.maxSetupMinutes !== undefined) {
conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) <= ?");
params.push(filters.maxSetupMinutes);
}
if (filters.minSetupMinutes !== undefined) {
conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) >= ?");
params.push(filters.minSetupMinutes);
}
if (filters.requiredService) {
conditions.push("json_extract(metadata_json, '$.required_services') LIKE ?");
params.push(`%"${filters.requiredService}"%`);
}
if (filters.targetAudience) {
conditions.push("json_extract(metadata_json, '$.target_audience') LIKE ?");
params.push(`%"${filters.targetAudience}"%`);
}
const query = `
SELECT * FROM templates
WHERE ${conditions.join(' AND ')}
ORDER BY views DESC, created_at DESC
LIMIT ? OFFSET ?
`;
params.push(limit, offset);
const results = this.db.prepare(query).all(...params) as StoredTemplate[];
logger.debug(`Metadata search found ${results.length} results`, { filters, count: results.length });
return results.map(t => this.decompressWorkflow(t));
}
/**
* Get count for metadata search results
*/
getMetadataSearchCount(filters: {
category?: string;
complexity?: 'simple' | 'medium' | 'complex';
maxSetupMinutes?: number;
minSetupMinutes?: number;
requiredService?: string;
targetAudience?: string;
}): number {
const conditions: string[] = ['metadata_json IS NOT NULL'];
const params: any[] = [];
if (filters.category) {
conditions.push("json_extract(metadata_json, '$.categories') LIKE ?");
params.push(`%"${filters.category}"%`);
}
if (filters.complexity) {
conditions.push("json_extract(metadata_json, '$.complexity') = ?");
params.push(filters.complexity);
}
if (filters.maxSetupMinutes !== undefined) {
conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) <= ?");
params.push(filters.maxSetupMinutes);
}
if (filters.minSetupMinutes !== undefined) {
conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) >= ?");
params.push(filters.minSetupMinutes);
}
if (filters.requiredService) {
conditions.push("json_extract(metadata_json, '$.required_services') LIKE ?");
params.push(`%"${filters.requiredService}"%`);
}
if (filters.targetAudience) {
conditions.push("json_extract(metadata_json, '$.target_audience') LIKE ?");
params.push(`%"${filters.targetAudience}"%`);
}
const query = `SELECT COUNT(*) as count FROM templates WHERE ${conditions.join(' AND ')}`;
const result = this.db.prepare(query).get(...params) as { count: number };
return result.count;
}
/**
* Get unique categories from metadata
*/
getAvailableCategories(): string[] {
const results = this.db.prepare(`
SELECT DISTINCT json_extract(value, '$') as category
FROM templates, json_each(json_extract(metadata_json, '$.categories'))
WHERE metadata_json IS NOT NULL
ORDER BY category
`).all() as { category: string }[];
return results.map(r => r.category);
}
/**
* Get unique target audiences from metadata
*/
getAvailableTargetAudiences(): string[] {
const results = this.db.prepare(`
SELECT DISTINCT json_extract(value, '$') as audience
FROM templates, json_each(json_extract(metadata_json, '$.target_audience'))
WHERE metadata_json IS NOT NULL
ORDER BY audience
`).all() as { audience: string }[];
return results.map(r => r.audience);
}
/**
* Get templates by category with metadata
*/
getTemplatesByCategory(category: string, limit: number = 10, offset: number = 0): StoredTemplate[] {
const query = `
SELECT * FROM templates
WHERE metadata_json IS NOT NULL
AND json_extract(metadata_json, '$.categories') LIKE ?
ORDER BY views DESC, created_at DESC
LIMIT ? OFFSET ?
`;
const results = this.db.prepare(query).all(`%"${category}"%`, limit, offset) as StoredTemplate[];
return results.map(t => this.decompressWorkflow(t));
}
/**
* Get templates by complexity level
*/
getTemplatesByComplexity(complexity: 'simple' | 'medium' | 'complex', limit: number = 10, offset: number = 0): StoredTemplate[] {
const query = `
SELECT * FROM templates
WHERE metadata_json IS NOT NULL
AND json_extract(metadata_json, '$.complexity') = ?
ORDER BY views DESC, created_at DESC
LIMIT ? OFFSET ?
`;
const results = this.db.prepare(query).all(complexity, limit, offset) as StoredTemplate[];
return results.map(t => this.decompressWorkflow(t));
}
}

141
src/templates/template-service.ts
View File

@@ -15,6 +15,15 @@ export interface TemplateInfo {
views: number;
created: string;
url: string;
metadata?: {
categories: string[];
complexity: 'simple' | 'medium' | 'complex';
use_cases: string[];
estimated_setup_minutes: number;
required_services: string[];
key_features: string[];
target_audience: string[];
};
}
export interface TemplateWithWorkflow extends TemplateInfo {
@@ -32,8 +41,18 @@ export interface PaginatedResponse<T> {
export interface TemplateMinimal {
id: number;
name: string;
description: string;
views: number;
nodeCount: number;
metadata?: {
categories: string[];
complexity: 'simple' | 'medium' | 'complex';
use_cases: string[];
estimated_setup_minutes: number;
required_services: string[];
key_features: string[];
target_audience: string[];
};
}
export class TemplateService {
@@ -137,16 +156,30 @@ export class TemplateService {
/**
* List all templates with minimal data
*/
async listTemplates(limit: number = 10, offset: number = 0, sortBy: 'views' | 'created_at' | 'name' = 'views'): Promise<PaginatedResponse<TemplateMinimal>> {
async listTemplates(limit: number = 10, offset: number = 0, sortBy: 'views' | 'created_at' | 'name' = 'views', includeMetadata: boolean = false): Promise<PaginatedResponse<TemplateMinimal>> {
const templates = this.repository.getAllTemplates(limit, offset, sortBy);
const total = this.repository.getTemplateCount();
const items = templates.map(t => ({
id: t.id,
name: t.name,
views: t.views,
nodeCount: JSON.parse(t.nodes_used).length
}));
const items = templates.map(t => {
const item: TemplateMinimal = {
id: t.id,
name: t.name,
description: t.description, // Always include description
views: t.views,
nodeCount: JSON.parse(t.nodes_used).length
};
// Optionally include metadata
if (includeMetadata && t.metadata_json) {
try {
item.metadata = JSON.parse(t.metadata_json);
} catch (error) {
logger.warn(`Failed to parse metadata for template ${t.id}:`, error);
}
}
return item;
});
return {
items,
@@ -175,6 +208,87 @@ export class TemplateService {
];
}
/**
* Search templates by metadata filters
*/
async searchTemplatesByMetadata(
filters: {
category?: string;
complexity?: 'simple' | 'medium' | 'complex';
maxSetupMinutes?: number;
minSetupMinutes?: number;
requiredService?: string;
targetAudience?: string;
},
limit: number = 20,
offset: number = 0
): Promise<PaginatedResponse<TemplateInfo>> {
const templates = this.repository.searchTemplatesByMetadata(filters, limit, offset);
const total = this.repository.getMetadataSearchCount(filters);
return {
items: templates.map(this.formatTemplateInfo.bind(this)),
total,
limit,
offset,
hasMore: offset + limit < total
};
}
/**
* Get available categories from template metadata
*/
async getAvailableCategories(): Promise<string[]> {
return this.repository.getAvailableCategories();
}
/**
* Get available target audiences from template metadata
*/
async getAvailableTargetAudiences(): Promise<string[]> {
return this.repository.getAvailableTargetAudiences();
}
/**
* Get templates by category
*/
async getTemplatesByCategory(
category: string,
limit: number = 10,
offset: number = 0
): Promise<PaginatedResponse<TemplateInfo>> {
const templates = this.repository.getTemplatesByCategory(category, limit, offset);
const total = this.repository.getMetadataSearchCount({ category });
return {
items: templates.map(this.formatTemplateInfo.bind(this)),
total,
limit,
offset,
hasMore: offset + limit < total
};
}
/**
* Get templates by complexity level
*/
async getTemplatesByComplexity(
complexity: 'simple' | 'medium' | 'complex',
limit: number = 10,
offset: number = 0
): Promise<PaginatedResponse<TemplateInfo>> {
const templates = this.repository.getTemplatesByComplexity(complexity, limit, offset);
const total = this.repository.getMetadataSearchCount({ complexity });
return {
items: templates.map(this.formatTemplateInfo.bind(this)),
total,
limit,
offset,
hasMore: offset + limit < total
};
}
/**
* Get template statistics
*/
@@ -263,7 +377,7 @@ export class TemplateService {
* Format stored template for API response
*/
private formatTemplateInfo(template: StoredTemplate): TemplateInfo {
return {
const info: TemplateInfo = {
id: template.id,
name: template.name,
description: template.description,
@@ -277,5 +391,16 @@ export class TemplateService {
created: template.created_at,
url: template.url
};
// Include metadata if available
if (template.metadata_json) {
try {
info.metadata = JSON.parse(template.metadata_json);
} catch (error) {
logger.warn(`Failed to parse metadata for template ${template.id}:`, error);
}
}
return info;
}
}