feat: add n8n workflow templates as MCP tools
- Add 4 new MCP tools for workflow templates - Integrate with n8n.io API to fetch community templates - Filter templates to last 6 months only - Store templates in SQLite with full workflow JSON - Manual fetch system (not part of regular rebuild) - Support search by nodes, keywords, and task categories - Add fetch:templates and test:templates npm scripts - Update to v2.4.1 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -21,4 +21,36 @@ CREATE TABLE IF NOT EXISTS nodes (
|
||||
-- Minimal indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_package ON nodes(package_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_ai_tool ON nodes(is_ai_tool);
|
||||
CREATE INDEX IF NOT EXISTS idx_category ON nodes(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_category ON nodes(category);
|
||||
|
||||
-- Templates table for n8n workflow templates
|
||||
CREATE TABLE IF NOT EXISTS templates (
|
||||
id INTEGER PRIMARY KEY,
|
||||
workflow_id INTEGER UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
author_name TEXT,
|
||||
author_username TEXT,
|
||||
author_verified INTEGER DEFAULT 0,
|
||||
nodes_used TEXT, -- JSON array of node types
|
||||
workflow_json TEXT NOT NULL, -- Complete workflow JSON
|
||||
categories TEXT, -- JSON array of categories
|
||||
views INTEGER DEFAULT 0,
|
||||
created_at DATETIME,
|
||||
updated_at DATETIME,
|
||||
url TEXT,
|
||||
scraped_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fresh_template CHECK (
|
||||
datetime(updated_at) >= datetime('now', '-6 months')
|
||||
)
|
||||
);
|
||||
|
||||
-- Templates indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_template_nodes ON templates(nodes_used);
|
||||
CREATE INDEX IF NOT EXISTS idx_template_updated ON templates(updated_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_template_name ON templates(name);
|
||||
|
||||
-- Full-text search for templates
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5(
|
||||
name, description, content=templates
|
||||
);
|
||||
@@ -17,6 +17,7 @@ import { TaskTemplates } from '../services/task-templates';
|
||||
import { ConfigValidator } from '../services/config-validator';
|
||||
import { PropertyDependencies } from '../services/property-dependencies';
|
||||
import { SimpleCache } from '../utils/simple-cache';
|
||||
import { TemplateService } from '../templates/template-service';
|
||||
|
||||
interface NodeRow {
|
||||
node_type: string;
|
||||
@@ -40,6 +41,7 @@ export class N8NDocumentationMCPServer {
|
||||
private server: Server;
|
||||
private db: DatabaseAdapter | null = null;
|
||||
private repository: NodeRepository | null = null;
|
||||
private templateService: TemplateService | null = null;
|
||||
private initialized: Promise<void>;
|
||||
private cache = new SimpleCache();
|
||||
|
||||
@@ -88,6 +90,7 @@ export class N8NDocumentationMCPServer {
|
||||
try {
|
||||
this.db = await createDatabaseAdapter(dbPath);
|
||||
this.repository = new NodeRepository(this.db);
|
||||
this.templateService = new TemplateService(this.db);
|
||||
logger.info(`Initialized database from: ${dbPath}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize database:', error);
|
||||
@@ -188,6 +191,14 @@ export class N8NDocumentationMCPServer {
|
||||
return this.validateNodeConfig(args.nodeType, args.config);
|
||||
case 'get_property_dependencies':
|
||||
return this.getPropertyDependencies(args.nodeType, args.config);
|
||||
case 'list_node_templates':
|
||||
return this.listNodeTemplates(args.nodeTypes, args.limit);
|
||||
case 'get_template':
|
||||
return this.getTemplate(args.templateId);
|
||||
case 'search_templates':
|
||||
return this.searchTemplates(args.query, args.limit);
|
||||
case 'get_templates_for_task':
|
||||
return this.getTemplatesForTask(args.task);
|
||||
default:
|
||||
throw new Error(`Unknown tool: ${name}`);
|
||||
}
|
||||
@@ -927,6 +938,108 @@ Full documentation is being prepared. For now, use get_node_essentials for confi
|
||||
transportType: transport.constructor.name
|
||||
});
|
||||
}
|
||||
|
||||
// Template-related methods
|
||||
private async listNodeTemplates(nodeTypes: string[], limit: number = 10): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.templateService) throw new Error('Template service not initialized');
|
||||
|
||||
const templates = await this.templateService.listNodeTemplates(nodeTypes, limit);
|
||||
|
||||
if (templates.length === 0) {
|
||||
return {
|
||||
message: `No templates found using nodes: ${nodeTypes.join(', ')}`,
|
||||
tip: "Try searching with more common nodes or run 'npm run fetch:templates' to update template database",
|
||||
templates: []
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
templates,
|
||||
count: templates.length,
|
||||
tip: `Use get_template(templateId) to get the full workflow JSON for any template`
|
||||
};
|
||||
}
|
||||
|
||||
private async getTemplate(templateId: number): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.templateService) throw new Error('Template service not initialized');
|
||||
|
||||
const template = await this.templateService.getTemplate(templateId);
|
||||
|
||||
if (!template) {
|
||||
return {
|
||||
error: `Template ${templateId} not found`,
|
||||
tip: "Use list_node_templates or search_templates to find available templates"
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
template,
|
||||
usage: "Import this workflow JSON directly into n8n or use it as a reference for building workflows"
|
||||
};
|
||||
}
|
||||
|
||||
private async searchTemplates(query: string, limit: number = 20): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.templateService) throw new Error('Template service not initialized');
|
||||
|
||||
const templates = await this.templateService.searchTemplates(query, limit);
|
||||
|
||||
if (templates.length === 0) {
|
||||
return {
|
||||
message: `No templates found matching: "${query}"`,
|
||||
tip: "Try different keywords or run 'npm run fetch:templates' to update template database",
|
||||
templates: []
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
templates,
|
||||
count: templates.length,
|
||||
query
|
||||
};
|
||||
}
|
||||
|
||||
private async getTemplatesForTask(task: string): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.templateService) throw new Error('Template service not initialized');
|
||||
|
||||
const templates = await this.templateService.getTemplatesForTask(task);
|
||||
const availableTasks = this.templateService.listAvailableTasks();
|
||||
|
||||
if (templates.length === 0) {
|
||||
return {
|
||||
message: `No templates found for task: ${task}`,
|
||||
availableTasks,
|
||||
tip: "Try a different task or use search_templates for custom searches"
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
task,
|
||||
templates,
|
||||
count: templates.length,
|
||||
description: this.getTaskDescription(task)
|
||||
};
|
||||
}
|
||||
|
||||
private getTaskDescription(task: string): string {
|
||||
const descriptions: Record<string, string> = {
|
||||
'ai_automation': 'AI-powered workflows using OpenAI, LangChain, and other AI tools',
|
||||
'data_sync': 'Synchronize data between databases, spreadsheets, and APIs',
|
||||
'webhook_processing': 'Process incoming webhooks and trigger automated actions',
|
||||
'email_automation': 'Send, receive, and process emails automatically',
|
||||
'slack_integration': 'Integrate with Slack for notifications and bot interactions',
|
||||
'data_transformation': 'Transform, clean, and manipulate data',
|
||||
'file_processing': 'Handle file uploads, downloads, and transformations',
|
||||
'scheduling': 'Schedule recurring tasks and time-based automations',
|
||||
'api_integration': 'Connect to external APIs and web services',
|
||||
'database_operations': 'Query, insert, update, and manage database records'
|
||||
};
|
||||
|
||||
return descriptions[task] || 'Workflow templates for this task';
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
// Ensure database is initialized before starting server
|
||||
|
||||
@@ -215,6 +215,85 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
|
||||
required: ['nodeType'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'list_node_templates',
|
||||
description: `List workflow templates that use specific node type(s). Returns ready-to-use workflows from n8n.io community. Templates are from the last 6 months only. Use node types like "n8n-nodes-base.httpRequest" or "@n8n/n8n-nodes-langchain.openAi". Great for finding proven workflow patterns.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
nodeTypes: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description: 'Array of node types to search for (e.g., ["n8n-nodes-base.httpRequest", "n8n-nodes-base.openAi"])',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
description: 'Maximum number of templates to return. Default 10.',
|
||||
default: 10,
|
||||
},
|
||||
},
|
||||
required: ['nodeTypes'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'get_template',
|
||||
description: `Get a specific workflow template with complete JSON. Returns the full workflow definition ready to import into n8n. Use template IDs from list_node_templates or search_templates results.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
templateId: {
|
||||
type: 'number',
|
||||
description: 'The template ID to retrieve',
|
||||
},
|
||||
},
|
||||
required: ['templateId'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'search_templates',
|
||||
description: `Search workflow templates by keywords in name/description. Returns templates matching your search terms. All templates are from the last 6 months and include view counts to gauge popularity. Good for finding workflows for specific use cases.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
query: {
|
||||
type: 'string',
|
||||
description: 'Search query (searches in template names and descriptions)',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
description: 'Maximum number of results. Default 20.',
|
||||
default: 20,
|
||||
},
|
||||
},
|
||||
required: ['query'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'get_templates_for_task',
|
||||
description: `Get recommended templates for common automation tasks. Returns curated templates that solve specific use cases. Available tasks: ai_automation, data_sync, webhook_processing, email_automation, slack_integration, data_transformation, file_processing, scheduling, api_integration, database_operations.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
task: {
|
||||
type: 'string',
|
||||
enum: [
|
||||
'ai_automation',
|
||||
'data_sync',
|
||||
'webhook_processing',
|
||||
'email_automation',
|
||||
'slack_integration',
|
||||
'data_transformation',
|
||||
'file_processing',
|
||||
'scheduling',
|
||||
'api_integration',
|
||||
'database_operations'
|
||||
],
|
||||
description: 'The type of task to get templates for',
|
||||
},
|
||||
},
|
||||
required: ['task'],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
75
src/scripts/fetch-templates.ts
Normal file
75
src/scripts/fetch-templates.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env node
|
||||
import { createDatabaseAdapter } from '../database/database-adapter';
|
||||
import { TemplateService } from '../templates/template-service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
async function fetchTemplates() {
|
||||
console.log('🌐 Fetching n8n workflow templates...\n');
|
||||
|
||||
// Ensure data directory exists
|
||||
const dataDir = './data';
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Initialize database
|
||||
const db = await createDatabaseAdapter('./data/nodes.db');
|
||||
|
||||
// Apply schema if needed
|
||||
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
|
||||
db.exec(schema);
|
||||
|
||||
// Create service
|
||||
const service = new TemplateService(db);
|
||||
|
||||
// Progress tracking
|
||||
let lastMessage = '';
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await service.fetchAndUpdateTemplates((message, current, total) => {
|
||||
// Clear previous line
|
||||
if (lastMessage) {
|
||||
process.stdout.write('\r' + ' '.repeat(lastMessage.length) + '\r');
|
||||
}
|
||||
|
||||
const progress = Math.round((current / total) * 100);
|
||||
lastMessage = `📊 ${message}: ${current}/${total} (${progress}%)`;
|
||||
process.stdout.write(lastMessage);
|
||||
});
|
||||
|
||||
console.log('\n'); // New line after progress
|
||||
|
||||
// Get stats
|
||||
const stats = await service.getTemplateStats();
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
console.log('✅ Template fetch complete!\n');
|
||||
console.log('📈 Statistics:');
|
||||
console.log(` - Total templates: ${stats.totalTemplates}`);
|
||||
console.log(` - Average views: ${stats.averageViews}`);
|
||||
console.log(` - Time elapsed: ${elapsed} seconds`);
|
||||
console.log('\n🔝 Top used nodes:');
|
||||
|
||||
stats.topUsedNodes.forEach((node: any, index: number) => {
|
||||
console.log(` ${index + 1}. ${node.node} (${node.count} templates)`);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Error fetching templates:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Close database
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
fetchTemplates().catch(console.error);
|
||||
}
|
||||
|
||||
export { fetchTemplates };
|
||||
88
src/scripts/test-templates.ts
Normal file
88
src/scripts/test-templates.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env node
|
||||
import { createDatabaseAdapter } from '../database/database-adapter';
|
||||
import { TemplateService } from '../templates/template-service';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
async function testTemplates() {
|
||||
console.log('🧪 Testing template functionality...\n');
|
||||
|
||||
// Initialize database
|
||||
const db = await createDatabaseAdapter('./data/nodes.db');
|
||||
|
||||
// Apply schema if needed
|
||||
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
|
||||
db.exec(schema);
|
||||
|
||||
// Create service
|
||||
const service = new TemplateService(db);
|
||||
|
||||
try {
|
||||
// Get statistics
|
||||
const stats = await service.getTemplateStats();
|
||||
console.log('📊 Template Database Stats:');
|
||||
console.log(` Total templates: ${stats.totalTemplates}`);
|
||||
|
||||
if (stats.totalTemplates === 0) {
|
||||
console.log('\n⚠️ No templates found in database!');
|
||||
console.log(' Run "npm run fetch:templates" to populate the database.\n');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(` Average views: ${stats.averageViews}`);
|
||||
console.log('\n🔝 Most used nodes in templates:');
|
||||
stats.topUsedNodes.forEach((node: any, i: number) => {
|
||||
console.log(` ${i + 1}. ${node.node} (${node.count} templates)`);
|
||||
});
|
||||
|
||||
// Test search
|
||||
console.log('\n🔍 Testing search for "webhook":');
|
||||
const searchResults = await service.searchTemplates('webhook', 3);
|
||||
searchResults.forEach((t: any) => {
|
||||
console.log(` - ${t.name} (${t.views} views)`);
|
||||
});
|
||||
|
||||
// Test node-based search
|
||||
console.log('\n🔍 Testing templates with HTTP Request node:');
|
||||
const httpTemplates = await service.listNodeTemplates(['n8n-nodes-base.httpRequest'], 3);
|
||||
httpTemplates.forEach((t: any) => {
|
||||
console.log(` - ${t.name} (${t.nodes.length} nodes)`);
|
||||
});
|
||||
|
||||
// Test task-based search
|
||||
console.log('\n🔍 Testing AI automation templates:');
|
||||
const aiTemplates = await service.getTemplatesForTask('ai_automation');
|
||||
aiTemplates.forEach((t: any) => {
|
||||
console.log(` - ${t.name} by @${t.author.username}`);
|
||||
});
|
||||
|
||||
// Get a specific template
|
||||
if (searchResults.length > 0) {
|
||||
const templateId = searchResults[0].id;
|
||||
console.log(`\n📄 Getting template ${templateId} details...`);
|
||||
const template = await service.getTemplate(templateId);
|
||||
if (template) {
|
||||
console.log(` Name: ${template.name}`);
|
||||
console.log(` Nodes: ${template.nodes.join(', ')}`);
|
||||
console.log(` Workflow has ${template.workflow.nodes.length} nodes`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n✅ All template tests passed!');
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error during testing:', error);
|
||||
}
|
||||
|
||||
// Close database
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
testTemplates().catch(console.error);
|
||||
}
|
||||
|
||||
export { testTemplates };
|
||||
86
src/templates/README.md
Normal file
86
src/templates/README.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# n8n Templates Integration
|
||||
|
||||
This module provides integration with n8n.io's workflow templates, allowing AI agents to discover and use proven workflow patterns.
|
||||
|
||||
## Features
|
||||
|
||||
- **API Integration**: Connects to n8n.io's official template API
|
||||
- **Fresh Templates**: Only includes templates updated within the last 6 months
|
||||
- **Manual Fetch**: Templates are fetched separately from the main node database
|
||||
- **Full Workflow JSON**: Complete workflow definitions ready for import
|
||||
- **Smart Search**: Find templates by nodes, keywords, or task categories
|
||||
|
||||
## Usage
|
||||
|
||||
### Fetching Templates
|
||||
|
||||
```bash
|
||||
npm run fetch:templates
|
||||
```
|
||||
|
||||
This command will:
|
||||
1. Connect to n8n.io API
|
||||
2. Fetch all templates from the last 6 months
|
||||
3. Download complete workflow JSON for each template
|
||||
4. Store in local SQLite database
|
||||
5. Display progress and statistics
|
||||
|
||||
### Testing
|
||||
|
||||
```bash
|
||||
npm run test:templates
|
||||
```
|
||||
|
||||
### MCP Tools
|
||||
|
||||
The following tools are available via MCP:
|
||||
|
||||
- `list_node_templates(nodeTypes, limit)` - Find templates using specific nodes
|
||||
- `get_template(templateId)` - Get complete workflow JSON
|
||||
- `search_templates(query, limit)` - Search by keywords
|
||||
- `get_templates_for_task(task)` - Get templates for common tasks
|
||||
|
||||
### Task Categories
|
||||
|
||||
- `ai_automation` - AI-powered workflows
|
||||
- `data_sync` - Database and spreadsheet synchronization
|
||||
- `webhook_processing` - Webhook handling workflows
|
||||
- `email_automation` - Email processing workflows
|
||||
- `slack_integration` - Slack bots and notifications
|
||||
- `data_transformation` - Data manipulation workflows
|
||||
- `file_processing` - File handling workflows
|
||||
- `scheduling` - Scheduled and recurring tasks
|
||||
- `api_integration` - External API connections
|
||||
- `database_operations` - Database CRUD operations
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Architecture
|
||||
|
||||
- `template-fetcher.ts` - Handles API communication and rate limiting
|
||||
- `template-repository.ts` - Database operations and queries
|
||||
- `template-service.ts` - Business logic and MCP integration
|
||||
|
||||
### Database Schema
|
||||
|
||||
Templates are stored in a dedicated table with:
|
||||
- Workflow metadata (name, description, author)
|
||||
- Node usage tracking
|
||||
- View counts for popularity
|
||||
- Complete workflow JSON
|
||||
- Creation/update timestamps
|
||||
- 6-month freshness constraint
|
||||
|
||||
### API Endpoints Used
|
||||
|
||||
- `/api/templates/workflows` - List all workflows
|
||||
- `/api/templates/search` - Search with pagination
|
||||
- `/api/templates/workflows/{id}` - Get specific workflow
|
||||
- `/api/templates/search/filters` - Available filters
|
||||
|
||||
## Notes
|
||||
|
||||
- Templates are NOT fetched during regular database rebuilds
|
||||
- Run `fetch:templates` manually when you need fresh templates
|
||||
- API rate limiting is implemented (200-500ms between requests)
|
||||
- Progress is shown during fetching for large datasets
|
||||
149
src/templates/template-fetcher.ts
Normal file
149
src/templates/template-fetcher.ts
Normal file
@@ -0,0 +1,149 @@
|
||||
import axios from 'axios';
|
||||
import { logger } from '../utils/logger';
|
||||
|
||||
export interface TemplateNode {
|
||||
id: number;
|
||||
name: string;
|
||||
icon: string;
|
||||
}
|
||||
|
||||
export interface TemplateUser {
|
||||
id: number;
|
||||
name: string;
|
||||
username: string;
|
||||
verified: boolean;
|
||||
}
|
||||
|
||||
export interface TemplateWorkflow {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string;
|
||||
totalViews: number;
|
||||
createdAt: string;
|
||||
user: TemplateUser;
|
||||
nodes: TemplateNode[];
|
||||
}
|
||||
|
||||
export interface TemplateDetail {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string;
|
||||
views: number;
|
||||
createdAt: string;
|
||||
workflow: {
|
||||
nodes: any[];
|
||||
connections: any;
|
||||
settings?: any;
|
||||
};
|
||||
}
|
||||
|
||||
export class TemplateFetcher {
|
||||
private readonly baseUrl = 'https://api.n8n.io/api/templates';
|
||||
private readonly pageSize = 100;
|
||||
|
||||
async fetchTemplates(progressCallback?: (current: number, total: number) => void): Promise<TemplateWorkflow[]> {
|
||||
const sixMonthsAgo = new Date();
|
||||
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
|
||||
|
||||
const allTemplates: TemplateWorkflow[] = [];
|
||||
let page = 1;
|
||||
let hasMore = true;
|
||||
|
||||
logger.info('Starting template fetch from n8n.io API');
|
||||
|
||||
while (hasMore) {
|
||||
try {
|
||||
const response = await axios.get(`${this.baseUrl}/search`, {
|
||||
params: {
|
||||
page,
|
||||
rows: this.pageSize,
|
||||
sort_by: 'last-updated'
|
||||
}
|
||||
});
|
||||
|
||||
const { workflows, totalWorkflows } = response.data;
|
||||
|
||||
// Filter templates by date
|
||||
const recentTemplates = workflows.filter((w: TemplateWorkflow) => {
|
||||
const createdDate = new Date(w.createdAt);
|
||||
return createdDate >= sixMonthsAgo;
|
||||
});
|
||||
|
||||
// If we hit templates older than 6 months, stop fetching
|
||||
if (recentTemplates.length < workflows.length) {
|
||||
hasMore = false;
|
||||
logger.info(`Reached templates older than 6 months at page ${page}`);
|
||||
}
|
||||
|
||||
allTemplates.push(...recentTemplates);
|
||||
|
||||
if (progressCallback) {
|
||||
progressCallback(allTemplates.length, Math.min(totalWorkflows, allTemplates.length + 500));
|
||||
}
|
||||
|
||||
// Check if there are more pages
|
||||
if (workflows.length < this.pageSize || allTemplates.length >= totalWorkflows) {
|
||||
hasMore = false;
|
||||
}
|
||||
|
||||
page++;
|
||||
|
||||
// Rate limiting - be nice to the API
|
||||
if (hasMore) {
|
||||
await this.sleep(500); // 500ms between requests
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error fetching templates page ${page}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Fetched ${allTemplates.length} templates from last 6 months`);
|
||||
return allTemplates;
|
||||
}
|
||||
|
||||
async fetchTemplateDetail(workflowId: number): Promise<TemplateDetail> {
|
||||
try {
|
||||
const response = await axios.get(`${this.baseUrl}/workflows/${workflowId}`);
|
||||
return response.data.workflow;
|
||||
} catch (error) {
|
||||
logger.error(`Error fetching template detail for ${workflowId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async fetchAllTemplateDetails(
|
||||
workflows: TemplateWorkflow[],
|
||||
progressCallback?: (current: number, total: number) => void
|
||||
): Promise<Map<number, TemplateDetail>> {
|
||||
const details = new Map<number, TemplateDetail>();
|
||||
|
||||
logger.info(`Fetching details for ${workflows.length} templates`);
|
||||
|
||||
for (let i = 0; i < workflows.length; i++) {
|
||||
const workflow = workflows[i];
|
||||
|
||||
try {
|
||||
const detail = await this.fetchTemplateDetail(workflow.id);
|
||||
details.set(workflow.id, detail);
|
||||
|
||||
if (progressCallback) {
|
||||
progressCallback(i + 1, workflows.length);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
await this.sleep(200); // 200ms between requests
|
||||
} catch (error) {
|
||||
logger.error(`Failed to fetch details for workflow ${workflow.id}:`, error);
|
||||
// Continue with other templates
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Successfully fetched ${details.size} template details`);
|
||||
return details;
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
181
src/templates/template-repository.ts
Normal file
181
src/templates/template-repository.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
import { DatabaseAdapter } from '../database/database-adapter';
|
||||
import { TemplateWorkflow, TemplateDetail } from './template-fetcher';
|
||||
import { logger } from '../utils/logger';
|
||||
|
||||
export interface StoredTemplate {
|
||||
id: number;
|
||||
workflow_id: number;
|
||||
name: string;
|
||||
description: string;
|
||||
author_name: string;
|
||||
author_username: string;
|
||||
author_verified: number;
|
||||
nodes_used: string; // JSON string
|
||||
workflow_json: string; // JSON string
|
||||
categories: string; // JSON string
|
||||
views: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
url: string;
|
||||
scraped_at: string;
|
||||
}
|
||||
|
||||
export class TemplateRepository {
|
||||
constructor(private db: DatabaseAdapter) {}
|
||||
|
||||
/**
|
||||
* Save a template to the database
|
||||
*/
|
||||
saveTemplate(workflow: TemplateWorkflow, detail: TemplateDetail, categories: string[] = []): void {
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT OR REPLACE INTO templates (
|
||||
id, workflow_id, name, description, author_name, author_username,
|
||||
author_verified, nodes_used, workflow_json, categories, views,
|
||||
created_at, updated_at, url
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
// Extract node types from workflow
|
||||
const nodeTypes = workflow.nodes.map(n => n.name);
|
||||
|
||||
// Build URL
|
||||
const url = `https://n8n.io/workflows/${workflow.id}`;
|
||||
|
||||
stmt.run(
|
||||
workflow.id,
|
||||
workflow.id,
|
||||
workflow.name,
|
||||
workflow.description || '',
|
||||
workflow.user.name,
|
||||
workflow.user.username,
|
||||
workflow.user.verified ? 1 : 0,
|
||||
JSON.stringify(nodeTypes),
|
||||
JSON.stringify(detail.workflow),
|
||||
JSON.stringify(categories),
|
||||
workflow.totalViews || 0,
|
||||
workflow.createdAt,
|
||||
workflow.createdAt, // Using createdAt as updatedAt since API doesn't provide updatedAt
|
||||
url
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get templates that use specific node types
|
||||
*/
|
||||
getTemplatesByNodes(nodeTypes: string[], limit: number = 10): StoredTemplate[] {
|
||||
// Build query for multiple node types
|
||||
const conditions = nodeTypes.map(() => "nodes_used LIKE ?").join(" OR ");
|
||||
const query = `
|
||||
SELECT * FROM templates
|
||||
WHERE ${conditions}
|
||||
ORDER BY views DESC, created_at DESC
|
||||
LIMIT ?
|
||||
`;
|
||||
|
||||
const params = [...nodeTypes.map(n => `%"${n}"%`), limit];
|
||||
return this.db.prepare(query).all(...params) as StoredTemplate[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific template by ID
|
||||
*/
|
||||
getTemplate(templateId: number): StoredTemplate | null {
|
||||
const row = this.db.prepare(`
|
||||
SELECT * FROM templates WHERE id = ?
|
||||
`).get(templateId) as StoredTemplate | undefined;
|
||||
|
||||
return row || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search templates by name or description
|
||||
*/
|
||||
searchTemplates(query: string, limit: number = 20): StoredTemplate[] {
|
||||
// Use FTS for search
|
||||
const ftsQuery = query.split(' ').map(term => `"${term}"`).join(' OR ');
|
||||
|
||||
return this.db.prepare(`
|
||||
SELECT t.* FROM templates t
|
||||
JOIN templates_fts ON t.id = templates_fts.rowid
|
||||
WHERE templates_fts MATCH ?
|
||||
ORDER BY rank, t.views DESC
|
||||
LIMIT ?
|
||||
`).all(ftsQuery, limit) as StoredTemplate[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get templates for a specific task/use case
|
||||
*/
|
||||
getTemplatesForTask(task: string): StoredTemplate[] {
|
||||
// Map tasks to relevant node combinations
|
||||
const taskNodeMap: Record<string, string[]> = {
|
||||
'ai_automation': ['@n8n/n8n-nodes-langchain.openAi', '@n8n/n8n-nodes-langchain.agent', 'n8n-nodes-base.openAi'],
|
||||
'data_sync': ['n8n-nodes-base.googleSheets', 'n8n-nodes-base.postgres', 'n8n-nodes-base.mysql'],
|
||||
'webhook_processing': ['n8n-nodes-base.webhook', 'n8n-nodes-base.httpRequest'],
|
||||
'email_automation': ['n8n-nodes-base.gmail', 'n8n-nodes-base.emailSend', 'n8n-nodes-base.emailReadImap'],
|
||||
'slack_integration': ['n8n-nodes-base.slack', 'n8n-nodes-base.slackTrigger'],
|
||||
'data_transformation': ['n8n-nodes-base.code', 'n8n-nodes-base.set', 'n8n-nodes-base.merge'],
|
||||
'file_processing': ['n8n-nodes-base.readBinaryFile', 'n8n-nodes-base.writeBinaryFile', 'n8n-nodes-base.googleDrive'],
|
||||
'scheduling': ['n8n-nodes-base.scheduleTrigger', 'n8n-nodes-base.cron'],
|
||||
'api_integration': ['n8n-nodes-base.httpRequest', 'n8n-nodes-base.graphql'],
|
||||
'database_operations': ['n8n-nodes-base.postgres', 'n8n-nodes-base.mysql', 'n8n-nodes-base.mongodb']
|
||||
};
|
||||
|
||||
const nodes = taskNodeMap[task];
|
||||
if (!nodes) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.getTemplatesByNodes(nodes, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total template count
|
||||
*/
|
||||
getTemplateCount(): number {
|
||||
const result = this.db.prepare('SELECT COUNT(*) as count FROM templates').get() as { count: number };
|
||||
return result.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get template statistics
|
||||
*/
|
||||
getTemplateStats(): Record<string, any> {
|
||||
const count = this.getTemplateCount();
|
||||
const avgViews = this.db.prepare('SELECT AVG(views) as avg FROM templates').get() as { avg: number };
|
||||
const topNodes = this.db.prepare(`
|
||||
SELECT nodes_used FROM templates
|
||||
ORDER BY views DESC
|
||||
LIMIT 100
|
||||
`).all() as { nodes_used: string }[];
|
||||
|
||||
// Count node usage
|
||||
const nodeCount: Record<string, number> = {};
|
||||
topNodes.forEach(t => {
|
||||
const nodes = JSON.parse(t.nodes_used);
|
||||
nodes.forEach((n: string) => {
|
||||
nodeCount[n] = (nodeCount[n] || 0) + 1;
|
||||
});
|
||||
});
|
||||
|
||||
// Get top 10 most used nodes
|
||||
const topUsedNodes = Object.entries(nodeCount)
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.slice(0, 10)
|
||||
.map(([node, count]) => ({ node, count }));
|
||||
|
||||
return {
|
||||
totalTemplates: count,
|
||||
averageViews: Math.round(avgViews.avg || 0),
|
||||
topUsedNodes
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all templates (for testing or refresh)
|
||||
*/
|
||||
clearTemplates(): void {
|
||||
this.db.exec('DELETE FROM templates');
|
||||
logger.info('Cleared all templates from database');
|
||||
}
|
||||
}
|
||||
160
src/templates/template-service.ts
Normal file
160
src/templates/template-service.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import { DatabaseAdapter } from '../database/database-adapter';
|
||||
import { TemplateRepository, StoredTemplate } from './template-repository';
|
||||
import { TemplateFetcher } from './template-fetcher';
|
||||
import { logger } from '../utils/logger';
|
||||
|
||||
export interface TemplateInfo {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string;
|
||||
author: {
|
||||
name: string;
|
||||
username: string;
|
||||
verified: boolean;
|
||||
};
|
||||
nodes: string[];
|
||||
views: number;
|
||||
created: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface TemplateWithWorkflow extends TemplateInfo {
|
||||
workflow: any;
|
||||
}
|
||||
|
||||
export class TemplateService {
|
||||
private repository: TemplateRepository;
|
||||
private fetcher: TemplateFetcher;
|
||||
|
||||
constructor(db: DatabaseAdapter) {
|
||||
this.repository = new TemplateRepository(db);
|
||||
this.fetcher = new TemplateFetcher();
|
||||
}
|
||||
|
||||
/**
|
||||
* List templates that use specific node types
|
||||
*/
|
||||
async listNodeTemplates(nodeTypes: string[], limit: number = 10): Promise<TemplateInfo[]> {
|
||||
const templates = this.repository.getTemplatesByNodes(nodeTypes, limit);
|
||||
return templates.map(this.formatTemplateInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific template with full workflow
|
||||
*/
|
||||
async getTemplate(templateId: number): Promise<TemplateWithWorkflow | null> {
|
||||
const template = this.repository.getTemplate(templateId);
|
||||
if (!template) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...this.formatTemplateInfo(template),
|
||||
workflow: JSON.parse(template.workflow_json)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search templates by query
|
||||
*/
|
||||
async searchTemplates(query: string, limit: number = 20): Promise<TemplateInfo[]> {
|
||||
const templates = this.repository.searchTemplates(query, limit);
|
||||
return templates.map(this.formatTemplateInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get templates for a specific task
|
||||
*/
|
||||
async getTemplatesForTask(task: string): Promise<TemplateInfo[]> {
|
||||
const templates = this.repository.getTemplatesForTask(task);
|
||||
return templates.map(this.formatTemplateInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* List available tasks
|
||||
*/
|
||||
listAvailableTasks(): string[] {
|
||||
return [
|
||||
'ai_automation',
|
||||
'data_sync',
|
||||
'webhook_processing',
|
||||
'email_automation',
|
||||
'slack_integration',
|
||||
'data_transformation',
|
||||
'file_processing',
|
||||
'scheduling',
|
||||
'api_integration',
|
||||
'database_operations'
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get template statistics
|
||||
*/
|
||||
async getTemplateStats(): Promise<Record<string, any>> {
|
||||
return this.repository.getTemplateStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and update templates from n8n.io
|
||||
*/
|
||||
async fetchAndUpdateTemplates(
|
||||
progressCallback?: (message: string, current: number, total: number) => void
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Clear existing templates
|
||||
this.repository.clearTemplates();
|
||||
|
||||
// Fetch template list
|
||||
logger.info('Fetching template list from n8n.io');
|
||||
const templates = await this.fetcher.fetchTemplates((current, total) => {
|
||||
progressCallback?.('Fetching template list', current, total);
|
||||
});
|
||||
|
||||
logger.info(`Found ${templates.length} templates from last 6 months`);
|
||||
|
||||
// Fetch details for each template
|
||||
logger.info('Fetching template details');
|
||||
const details = await this.fetcher.fetchAllTemplateDetails(templates, (current, total) => {
|
||||
progressCallback?.('Fetching template details', current, total);
|
||||
});
|
||||
|
||||
// Save to database
|
||||
logger.info('Saving templates to database');
|
||||
let saved = 0;
|
||||
for (const template of templates) {
|
||||
const detail = details.get(template.id);
|
||||
if (detail) {
|
||||
this.repository.saveTemplate(template, detail);
|
||||
saved++;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Successfully saved ${saved} templates to database`);
|
||||
progressCallback?.('Complete', saved, saved);
|
||||
} catch (error) {
|
||||
logger.error('Error fetching templates:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format stored template for API response
|
||||
*/
|
||||
private formatTemplateInfo(template: StoredTemplate): TemplateInfo {
|
||||
return {
|
||||
id: template.id,
|
||||
name: template.name,
|
||||
description: template.description,
|
||||
author: {
|
||||
name: template.author_name,
|
||||
username: template.author_username,
|
||||
verified: template.author_verified === 1
|
||||
},
|
||||
nodes: JSON.parse(template.nodes_used),
|
||||
views: template.views,
|
||||
created: template.created_at,
|
||||
url: template.url
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user