From 08f9d1ad307dea895ca8cababccc9c310143c09d Mon Sep 17 00:00:00 2001 From: czlonkowski <56956555+czlonkowski@users.noreply.github.com> Date: Fri, 20 Jun 2025 00:02:09 +0200 Subject: [PATCH] feat: add n8n workflow templates as MCP tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add 4 new MCP tools for workflow templates - Integrate with n8n.io API to fetch community templates - Filter templates to last 6 months only - Store templates in SQLite with full workflow JSON - Manual fetch system (not part of regular rebuild) - Support search by nodes, keywords, and task categories - Add fetch:templates and test:templates npm scripts - Update to v2.4.1 ๐Ÿค– Generated with Claude Code Co-Authored-By: Claude --- CLAUDE.md | 29 ++++- data/nodes.db | Bin 11997184 -> 12034048 bytes package-lock.json | 102 ++++++++++----- package.json | 5 +- src/database/schema.sql | 34 ++++- src/mcp/server-update.ts | 113 +++++++++++++++++ src/mcp/tools-update.ts | 79 ++++++++++++ src/scripts/fetch-templates.ts | 75 +++++++++++ src/scripts/test-templates.ts | 88 +++++++++++++ src/templates/README.md | 86 +++++++++++++ src/templates/template-fetcher.ts | 149 ++++++++++++++++++++++ src/templates/template-repository.ts | 181 +++++++++++++++++++++++++++ src/templates/template-service.ts | 160 +++++++++++++++++++++++ 13 files changed, 1068 insertions(+), 33 deletions(-) create mode 100644 src/scripts/fetch-templates.ts create mode 100644 src/scripts/test-templates.ts create mode 100644 src/templates/README.md create mode 100644 src/templates/template-fetcher.ts create mode 100644 src/templates/template-repository.ts create mode 100644 src/templates/template-service.ts diff --git a/CLAUDE.md b/CLAUDE.md index bbc2078..392b196 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,18 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co n8n-mcp is a comprehensive documentation and knowledge server that provides AI assistants with complete access to n8n node information through the Model Context Protocol (MCP). It serves as a bridge between n8n's workflow automation platform and AI models, enabling them to understand and work with n8n nodes effectively. -## โœ… Latest Updates (v2.4.0) +## โœ… Latest Updates (v2.4.1) + +### Update (v2.4.1) - n8n Workflow Templates: +- โœ… **NEW: list_node_templates tool** - Find workflow templates using specific nodes +- โœ… **NEW: get_template tool** - Get complete workflow JSON for import +- โœ… **NEW: search_templates tool** - Search templates by keywords +- โœ… **NEW: get_templates_for_task tool** - Get curated templates for common tasks +- โœ… Added Templates system with n8n.io API integration +- โœ… Templates filtered to last 6 months only (freshness guarantee) +- โœ… Manual fetch system - not part of regular rebuild +- โœ… Full workflow JSON available for immediate use +- โœ… 10 task categories: AI automation, data sync, webhooks, etc. ### Update (v2.4.0) - AI-Optimized MCP Tools: - โœ… **NEW: get_node_essentials tool** - Returns only 10-20 essential properties (95% size reduction) @@ -77,11 +88,17 @@ src/ โ”‚ โ”œโ”€โ”€ task-templates.ts # Pre-configured node settings (NEW in v2.4) โ”‚ โ”œโ”€โ”€ config-validator.ts # Configuration validation (NEW in v2.4) โ”‚ โ””โ”€โ”€ property-dependencies.ts # Dependency analysis (NEW in v2.4) +โ”œโ”€โ”€ templates/ +โ”‚ โ”œโ”€โ”€ template-fetcher.ts # Fetches templates from n8n.io API (NEW in v2.4.1) +โ”‚ โ”œโ”€โ”€ template-repository.ts # Template database operations (NEW in v2.4.1) +โ”‚ โ””โ”€โ”€ template-service.ts # Template business logic (NEW in v2.4.1) โ”œโ”€โ”€ scripts/ โ”‚ โ”œโ”€โ”€ rebuild.ts # Database rebuild with validation โ”‚ โ”œโ”€โ”€ validate.ts # Node validation โ”‚ โ”œโ”€โ”€ test-nodes.ts # Critical node tests -โ”‚ โ””โ”€โ”€ test-essentials.ts # Test new essentials tools (NEW in v2.4) +โ”‚ โ”œโ”€โ”€ test-essentials.ts # Test new essentials tools (NEW in v2.4) +โ”‚ โ”œโ”€โ”€ fetch-templates.ts # Fetch workflow templates from n8n.io (NEW in v2.4.1) +โ”‚ โ””โ”€โ”€ test-templates.ts # Test template functionality (NEW in v2.4.1) โ”œโ”€โ”€ mcp/ โ”‚ โ”œโ”€โ”€ server-update.ts # MCP server with enhanced tools โ”‚ โ”œโ”€โ”€ tools-update.ts # Tool definitions including new essentials @@ -119,6 +136,10 @@ npm run rebuild:optimized # Build database with embedded source code npm run validate # Validate critical nodes npm run test-nodes # Test critical node properties/operations +# Template Commands: +npm run fetch:templates # Fetch workflow templates from n8n.io (manual) +npm run test:templates # Test template functionality + # Dependency Update Commands: npm run update:n8n:check # Check for n8n updates (dry run) npm run update:n8n # Update n8n packages to latest versions @@ -229,6 +250,10 @@ The project implements MCP (Model Context Protocol) to expose n8n node documenta - `list_ai_tools` - List all AI-capable nodes (usableAsTool: true) - `get_node_documentation` - Get parsed documentation from n8n-docs - `get_database_statistics` - Get database usage statistics and metrics +- `list_node_templates` - **NEW** Find workflow templates using specific nodes +- `get_template` - **NEW** Get complete workflow JSON for import +- `search_templates` - **NEW** Search templates by keywords +- `get_templates_for_task` - **NEW** Get curated templates for common tasks ### Database Structure Uses SQLite with enhanced schema: diff --git a/data/nodes.db b/data/nodes.db index 2c7a7eeb74305e1b10f9a4b7de30980d8e49fc43..4e72a1132cbc2d112834fe135e8f01890014e9db 100644 GIT binary patch delta 2586 zcmb`}*>h7>9Ki7;rAeEmp`eCE?GcwGkWva3RIJh^Kp-R~O`vrPA-QcYO>;}`?W#3I z5X23zT6aZJ+!Z(a%1obq_rdWW@Btri#uDmtCUzNErr^VIOWXdns5(m?gCJhXx6Pl#Pdo(qkR5Cqs??Pj? z@wSM+CF&Q^me!zOxEG3Xd-_F3B(SF?(k*uTyM1EVEA|JXJHxT4h=lhC+9zr(uH2R_ zro3x`+$klg%HuZ`R2;O8-70%A$nB9+A`puDclaaab*ST_H5hL7=4}p3&ehOh%C{|0 zfSejFe=7!Tqqj;-fZL-=y>iMYG$}ja6N9}O#Wy$k+{a$M9l1IE7_nPijg5=6BcZV^ zb2lf&h;8In_2?0cci~`KN%o6ep%W7}i_7h<%5RB_(!!37Yhi4+WlQ(UZ4nvXne6r9(}4;s^)mtnexW@OAigwpq!MNonaeVeECAK zlq%0q9E>M4?|~|_tFg{BF+WX4;q^;7wDKIevX|9E%Tp9)$A?rYr=9Ytr1awS5$vk9HY>z-uIiwho!9CBh$Yd7qZ>Iy(JcmiVgbY(o0^SC+h3P zu1+>HkQk>1wui8qfbkWp!U?o;zHSWY+xEpm4xCd)+FYd$rcmQsA;Kf?3 zgAessj|OZ&BQ|0a9z+v1V+$U_!*~Q+@hF<{7#_zqw4fDjXonx$u>(61z%J}Y5PJ|p z7#-M)2s#l(4EwMjUFgOWcoI+HX*`2C4j_SpNFoIZJ?KRrWE?_2(ilJn3I=f)S*Xx3 zgkg+e6k{02vv>{@cpf<%K^`yQD30Mp9LEW~gp+s~ui#a@hSzZlr*Q^vU=nZQEllBU zyn}b~9^S_X_z)lAW1Ph&_!Q^x89v8(e1R|V6~4wd_!i&cd;EY4_z^$hXZ(U+adCRa zKHXd~bt%8ZWLP>~dtuw}j%m?hTw^j@_1{we&(M^xS6ZBx4bH31%cUGPlgnzYFsv&6 qW~(q(80_6|6|0SWvZ{QqF8L3H2Zeh8 delta 798 zcmYMq*H_H}0KoAdb*+B4>slduOER)aC|hL9-ZLY6yF}T_-m9x7}Sq(v6RZcYaCy0cpAX*>$87+%uhN)z{ zw7ev&Ooi2{un~<(Bb^9MXi77h(}E2C{ZTTB(ULf=XiXd1(vB?J(}9j;(}~V>p)1|! zP7iw0i{A91Fa79G4g<(#AbAX8FhdwhKEoK!2u3oB(Trg%<0xP}g-l=~lPIE?$xLA? z)0oZ-W-^NsW;2IU<}#1@EMOsvSj-a2SjsY%vx1eZVl``6%R1JxfsJfpGh5ioHp=18.0.0" } }, - "node_modules/@n8n/localtunnel/node_modules/axios": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz", - "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/@n8n/n8n-nodes-langchain": { "version": "1.97.0", "resolved": "https://registry.npmjs.org/@n8n/n8n-nodes-langchain/-/n8n-nodes-langchain-1.97.0.tgz", @@ -10533,6 +10534,17 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/@n8n/task-runner/node_modules/axios": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/@n8n/task-runner/node_modules/camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", @@ -12115,6 +12127,17 @@ "tslib": "2.6.2" } }, + "node_modules/@rudderstack/rudder-sdk-node/node_modules/axios": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/@rudderstack/rudder-sdk-node/node_modules/uuid": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.2.tgz", @@ -14298,9 +14321,9 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", - "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.10.0.tgz", + "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", @@ -18745,18 +18768,6 @@ "undici-types": "~5.26.4" } }, - "node_modules/ibm-cloud-sdk-core/node_modules/axios": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz", - "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", - "license": "MIT", - "peer": true, - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/ibm-cloud-sdk-core/node_modules/camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", @@ -22802,6 +22813,17 @@ "zod": "3.24.1" } }, + "node_modules/n8n-core/node_modules/axios": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/n8n-core/node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -23047,6 +23069,17 @@ "zod": "3.24.1" } }, + "node_modules/n8n-workflow/node_modules/axios": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/n8n-workflow/node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -26309,6 +26342,17 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, + "node_modules/n8n/node_modules/axios": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/n8n/node_modules/better-sqlite3": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-9.6.0.tgz", diff --git a/package.json b/package.json index 2fce553..071c33e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "n8n-mcp", - "version": "2.4.0", + "version": "2.4.1", "description": "Integration between n8n workflow automation and Model Context Protocol (MCP)", "main": "dist/index.js", "scripts": { @@ -22,6 +22,8 @@ "typecheck": "tsc --noEmit", "update:n8n": "node scripts/update-n8n-deps.js", "update:n8n:check": "node scripts/update-n8n-deps.js --dry-run", + "fetch:templates": "node dist/scripts/fetch-templates.js", + "test:templates": "node dist/scripts/test-templates.js", "db:rebuild": "node dist/scripts/rebuild-database.js", "db:init": "node -e \"new (require('./dist/services/sqlite-storage-service').SQLiteStorageService)(); console.log('Database initialized')\"", "docs:rebuild": "ts-node src/scripts/rebuild-database.ts" @@ -58,6 +60,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.12.1", "@n8n/n8n-nodes-langchain": "^1.96.1", + "axios": "^1.10.0", "better-sqlite3": "^11.10.0", "dotenv": "^16.5.0", "express": "^5.1.0", diff --git a/src/database/schema.sql b/src/database/schema.sql index c970123..2d0bb8e 100644 --- a/src/database/schema.sql +++ b/src/database/schema.sql @@ -21,4 +21,36 @@ CREATE TABLE IF NOT EXISTS nodes ( -- Minimal indexes for performance CREATE INDEX IF NOT EXISTS idx_package ON nodes(package_name); CREATE INDEX IF NOT EXISTS idx_ai_tool ON nodes(is_ai_tool); -CREATE INDEX IF NOT EXISTS idx_category ON nodes(category); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_category ON nodes(category); + +-- Templates table for n8n workflow templates +CREATE TABLE IF NOT EXISTS templates ( + id INTEGER PRIMARY KEY, + workflow_id INTEGER UNIQUE NOT NULL, + name TEXT NOT NULL, + description TEXT, + author_name TEXT, + author_username TEXT, + author_verified INTEGER DEFAULT 0, + nodes_used TEXT, -- JSON array of node types + workflow_json TEXT NOT NULL, -- Complete workflow JSON + categories TEXT, -- JSON array of categories + views INTEGER DEFAULT 0, + created_at DATETIME, + updated_at DATETIME, + url TEXT, + scraped_at DATETIME DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT fresh_template CHECK ( + datetime(updated_at) >= datetime('now', '-6 months') + ) +); + +-- Templates indexes +CREATE INDEX IF NOT EXISTS idx_template_nodes ON templates(nodes_used); +CREATE INDEX IF NOT EXISTS idx_template_updated ON templates(updated_at); +CREATE INDEX IF NOT EXISTS idx_template_name ON templates(name); + +-- Full-text search for templates +CREATE VIRTUAL TABLE IF NOT EXISTS templates_fts USING fts5( + name, description, content=templates +); \ No newline at end of file diff --git a/src/mcp/server-update.ts b/src/mcp/server-update.ts index 9be2a26..2b55c13 100644 --- a/src/mcp/server-update.ts +++ b/src/mcp/server-update.ts @@ -17,6 +17,7 @@ import { TaskTemplates } from '../services/task-templates'; import { ConfigValidator } from '../services/config-validator'; import { PropertyDependencies } from '../services/property-dependencies'; import { SimpleCache } from '../utils/simple-cache'; +import { TemplateService } from '../templates/template-service'; interface NodeRow { node_type: string; @@ -40,6 +41,7 @@ export class N8NDocumentationMCPServer { private server: Server; private db: DatabaseAdapter | null = null; private repository: NodeRepository | null = null; + private templateService: TemplateService | null = null; private initialized: Promise; private cache = new SimpleCache(); @@ -88,6 +90,7 @@ export class N8NDocumentationMCPServer { try { this.db = await createDatabaseAdapter(dbPath); this.repository = new NodeRepository(this.db); + this.templateService = new TemplateService(this.db); logger.info(`Initialized database from: ${dbPath}`); } catch (error) { logger.error('Failed to initialize database:', error); @@ -188,6 +191,14 @@ export class N8NDocumentationMCPServer { return this.validateNodeConfig(args.nodeType, args.config); case 'get_property_dependencies': return this.getPropertyDependencies(args.nodeType, args.config); + case 'list_node_templates': + return this.listNodeTemplates(args.nodeTypes, args.limit); + case 'get_template': + return this.getTemplate(args.templateId); + case 'search_templates': + return this.searchTemplates(args.query, args.limit); + case 'get_templates_for_task': + return this.getTemplatesForTask(args.task); default: throw new Error(`Unknown tool: ${name}`); } @@ -927,6 +938,108 @@ Full documentation is being prepared. For now, use get_node_essentials for confi transportType: transport.constructor.name }); } + + // Template-related methods + private async listNodeTemplates(nodeTypes: string[], limit: number = 10): Promise { + await this.ensureInitialized(); + if (!this.templateService) throw new Error('Template service not initialized'); + + const templates = await this.templateService.listNodeTemplates(nodeTypes, limit); + + if (templates.length === 0) { + return { + message: `No templates found using nodes: ${nodeTypes.join(', ')}`, + tip: "Try searching with more common nodes or run 'npm run fetch:templates' to update template database", + templates: [] + }; + } + + return { + templates, + count: templates.length, + tip: `Use get_template(templateId) to get the full workflow JSON for any template` + }; + } + + private async getTemplate(templateId: number): Promise { + await this.ensureInitialized(); + if (!this.templateService) throw new Error('Template service not initialized'); + + const template = await this.templateService.getTemplate(templateId); + + if (!template) { + return { + error: `Template ${templateId} not found`, + tip: "Use list_node_templates or search_templates to find available templates" + }; + } + + return { + template, + usage: "Import this workflow JSON directly into n8n or use it as a reference for building workflows" + }; + } + + private async searchTemplates(query: string, limit: number = 20): Promise { + await this.ensureInitialized(); + if (!this.templateService) throw new Error('Template service not initialized'); + + const templates = await this.templateService.searchTemplates(query, limit); + + if (templates.length === 0) { + return { + message: `No templates found matching: "${query}"`, + tip: "Try different keywords or run 'npm run fetch:templates' to update template database", + templates: [] + }; + } + + return { + templates, + count: templates.length, + query + }; + } + + private async getTemplatesForTask(task: string): Promise { + await this.ensureInitialized(); + if (!this.templateService) throw new Error('Template service not initialized'); + + const templates = await this.templateService.getTemplatesForTask(task); + const availableTasks = this.templateService.listAvailableTasks(); + + if (templates.length === 0) { + return { + message: `No templates found for task: ${task}`, + availableTasks, + tip: "Try a different task or use search_templates for custom searches" + }; + } + + return { + task, + templates, + count: templates.length, + description: this.getTaskDescription(task) + }; + } + + private getTaskDescription(task: string): string { + const descriptions: Record = { + 'ai_automation': 'AI-powered workflows using OpenAI, LangChain, and other AI tools', + 'data_sync': 'Synchronize data between databases, spreadsheets, and APIs', + 'webhook_processing': 'Process incoming webhooks and trigger automated actions', + 'email_automation': 'Send, receive, and process emails automatically', + 'slack_integration': 'Integrate with Slack for notifications and bot interactions', + 'data_transformation': 'Transform, clean, and manipulate data', + 'file_processing': 'Handle file uploads, downloads, and transformations', + 'scheduling': 'Schedule recurring tasks and time-based automations', + 'api_integration': 'Connect to external APIs and web services', + 'database_operations': 'Query, insert, update, and manage database records' + }; + + return descriptions[task] || 'Workflow templates for this task'; + } async run(): Promise { // Ensure database is initialized before starting server diff --git a/src/mcp/tools-update.ts b/src/mcp/tools-update.ts index 294b6ca..33a2480 100644 --- a/src/mcp/tools-update.ts +++ b/src/mcp/tools-update.ts @@ -215,6 +215,85 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [ required: ['nodeType'], }, }, + { + name: 'list_node_templates', + description: `List workflow templates that use specific node type(s). Returns ready-to-use workflows from n8n.io community. Templates are from the last 6 months only. Use node types like "n8n-nodes-base.httpRequest" or "@n8n/n8n-nodes-langchain.openAi". Great for finding proven workflow patterns.`, + inputSchema: { + type: 'object', + properties: { + nodeTypes: { + type: 'array', + items: { type: 'string' }, + description: 'Array of node types to search for (e.g., ["n8n-nodes-base.httpRequest", "n8n-nodes-base.openAi"])', + }, + limit: { + type: 'number', + description: 'Maximum number of templates to return. Default 10.', + default: 10, + }, + }, + required: ['nodeTypes'], + }, + }, + { + name: 'get_template', + description: `Get a specific workflow template with complete JSON. Returns the full workflow definition ready to import into n8n. Use template IDs from list_node_templates or search_templates results.`, + inputSchema: { + type: 'object', + properties: { + templateId: { + type: 'number', + description: 'The template ID to retrieve', + }, + }, + required: ['templateId'], + }, + }, + { + name: 'search_templates', + description: `Search workflow templates by keywords in name/description. Returns templates matching your search terms. All templates are from the last 6 months and include view counts to gauge popularity. Good for finding workflows for specific use cases.`, + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Search query (searches in template names and descriptions)', + }, + limit: { + type: 'number', + description: 'Maximum number of results. Default 20.', + default: 20, + }, + }, + required: ['query'], + }, + }, + { + name: 'get_templates_for_task', + description: `Get recommended templates for common automation tasks. Returns curated templates that solve specific use cases. Available tasks: ai_automation, data_sync, webhook_processing, email_automation, slack_integration, data_transformation, file_processing, scheduling, api_integration, database_operations.`, + inputSchema: { + type: 'object', + properties: { + task: { + type: 'string', + enum: [ + 'ai_automation', + 'data_sync', + 'webhook_processing', + 'email_automation', + 'slack_integration', + 'data_transformation', + 'file_processing', + 'scheduling', + 'api_integration', + 'database_operations' + ], + description: 'The type of task to get templates for', + }, + }, + required: ['task'], + }, + }, ]; /** diff --git a/src/scripts/fetch-templates.ts b/src/scripts/fetch-templates.ts new file mode 100644 index 0000000..e119ad5 --- /dev/null +++ b/src/scripts/fetch-templates.ts @@ -0,0 +1,75 @@ +#!/usr/bin/env node +import { createDatabaseAdapter } from '../database/database-adapter'; +import { TemplateService } from '../templates/template-service'; +import * as fs from 'fs'; +import * as path from 'path'; + +async function fetchTemplates() { + console.log('๐ŸŒ Fetching n8n workflow templates...\n'); + + // Ensure data directory exists + const dataDir = './data'; + if (!fs.existsSync(dataDir)) { + fs.mkdirSync(dataDir, { recursive: true }); + } + + // Initialize database + const db = await createDatabaseAdapter('./data/nodes.db'); + + // Apply schema if needed + const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8'); + db.exec(schema); + + // Create service + const service = new TemplateService(db); + + // Progress tracking + let lastMessage = ''; + const startTime = Date.now(); + + try { + await service.fetchAndUpdateTemplates((message, current, total) => { + // Clear previous line + if (lastMessage) { + process.stdout.write('\r' + ' '.repeat(lastMessage.length) + '\r'); + } + + const progress = Math.round((current / total) * 100); + lastMessage = `๐Ÿ“Š ${message}: ${current}/${total} (${progress}%)`; + process.stdout.write(lastMessage); + }); + + console.log('\n'); // New line after progress + + // Get stats + const stats = await service.getTemplateStats(); + const elapsed = Math.round((Date.now() - startTime) / 1000); + + console.log('โœ… Template fetch complete!\n'); + console.log('๐Ÿ“ˆ Statistics:'); + console.log(` - Total templates: ${stats.totalTemplates}`); + console.log(` - Average views: ${stats.averageViews}`); + console.log(` - Time elapsed: ${elapsed} seconds`); + console.log('\n๐Ÿ” Top used nodes:'); + + stats.topUsedNodes.forEach((node: any, index: number) => { + console.log(` ${index + 1}. ${node.node} (${node.count} templates)`); + }); + + } catch (error) { + console.error('\nโŒ Error fetching templates:', error); + process.exit(1); + } + + // Close database + if ('close' in db && typeof db.close === 'function') { + db.close(); + } +} + +// Run if called directly +if (require.main === module) { + fetchTemplates().catch(console.error); +} + +export { fetchTemplates }; \ No newline at end of file diff --git a/src/scripts/test-templates.ts b/src/scripts/test-templates.ts new file mode 100644 index 0000000..fc70e7f --- /dev/null +++ b/src/scripts/test-templates.ts @@ -0,0 +1,88 @@ +#!/usr/bin/env node +import { createDatabaseAdapter } from '../database/database-adapter'; +import { TemplateService } from '../templates/template-service'; +import * as fs from 'fs'; +import * as path from 'path'; + +async function testTemplates() { + console.log('๐Ÿงช Testing template functionality...\n'); + + // Initialize database + const db = await createDatabaseAdapter('./data/nodes.db'); + + // Apply schema if needed + const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8'); + db.exec(schema); + + // Create service + const service = new TemplateService(db); + + try { + // Get statistics + const stats = await service.getTemplateStats(); + console.log('๐Ÿ“Š Template Database Stats:'); + console.log(` Total templates: ${stats.totalTemplates}`); + + if (stats.totalTemplates === 0) { + console.log('\nโš ๏ธ No templates found in database!'); + console.log(' Run "npm run fetch:templates" to populate the database.\n'); + return; + } + + console.log(` Average views: ${stats.averageViews}`); + console.log('\n๐Ÿ” Most used nodes in templates:'); + stats.topUsedNodes.forEach((node: any, i: number) => { + console.log(` ${i + 1}. ${node.node} (${node.count} templates)`); + }); + + // Test search + console.log('\n๐Ÿ” Testing search for "webhook":'); + const searchResults = await service.searchTemplates('webhook', 3); + searchResults.forEach((t: any) => { + console.log(` - ${t.name} (${t.views} views)`); + }); + + // Test node-based search + console.log('\n๐Ÿ” Testing templates with HTTP Request node:'); + const httpTemplates = await service.listNodeTemplates(['n8n-nodes-base.httpRequest'], 3); + httpTemplates.forEach((t: any) => { + console.log(` - ${t.name} (${t.nodes.length} nodes)`); + }); + + // Test task-based search + console.log('\n๐Ÿ” Testing AI automation templates:'); + const aiTemplates = await service.getTemplatesForTask('ai_automation'); + aiTemplates.forEach((t: any) => { + console.log(` - ${t.name} by @${t.author.username}`); + }); + + // Get a specific template + if (searchResults.length > 0) { + const templateId = searchResults[0].id; + console.log(`\n๐Ÿ“„ Getting template ${templateId} details...`); + const template = await service.getTemplate(templateId); + if (template) { + console.log(` Name: ${template.name}`); + console.log(` Nodes: ${template.nodes.join(', ')}`); + console.log(` Workflow has ${template.workflow.nodes.length} nodes`); + } + } + + console.log('\nโœ… All template tests passed!'); + + } catch (error) { + console.error('โŒ Error during testing:', error); + } + + // Close database + if ('close' in db && typeof db.close === 'function') { + db.close(); + } +} + +// Run if called directly +if (require.main === module) { + testTemplates().catch(console.error); +} + +export { testTemplates }; \ No newline at end of file diff --git a/src/templates/README.md b/src/templates/README.md new file mode 100644 index 0000000..4e30cf9 --- /dev/null +++ b/src/templates/README.md @@ -0,0 +1,86 @@ +# n8n Templates Integration + +This module provides integration with n8n.io's workflow templates, allowing AI agents to discover and use proven workflow patterns. + +## Features + +- **API Integration**: Connects to n8n.io's official template API +- **Fresh Templates**: Only includes templates updated within the last 6 months +- **Manual Fetch**: Templates are fetched separately from the main node database +- **Full Workflow JSON**: Complete workflow definitions ready for import +- **Smart Search**: Find templates by nodes, keywords, or task categories + +## Usage + +### Fetching Templates + +```bash +npm run fetch:templates +``` + +This command will: +1. Connect to n8n.io API +2. Fetch all templates from the last 6 months +3. Download complete workflow JSON for each template +4. Store in local SQLite database +5. Display progress and statistics + +### Testing + +```bash +npm run test:templates +``` + +### MCP Tools + +The following tools are available via MCP: + +- `list_node_templates(nodeTypes, limit)` - Find templates using specific nodes +- `get_template(templateId)` - Get complete workflow JSON +- `search_templates(query, limit)` - Search by keywords +- `get_templates_for_task(task)` - Get templates for common tasks + +### Task Categories + +- `ai_automation` - AI-powered workflows +- `data_sync` - Database and spreadsheet synchronization +- `webhook_processing` - Webhook handling workflows +- `email_automation` - Email processing workflows +- `slack_integration` - Slack bots and notifications +- `data_transformation` - Data manipulation workflows +- `file_processing` - File handling workflows +- `scheduling` - Scheduled and recurring tasks +- `api_integration` - External API connections +- `database_operations` - Database CRUD operations + +## Implementation Details + +### Architecture + +- `template-fetcher.ts` - Handles API communication and rate limiting +- `template-repository.ts` - Database operations and queries +- `template-service.ts` - Business logic and MCP integration + +### Database Schema + +Templates are stored in a dedicated table with: +- Workflow metadata (name, description, author) +- Node usage tracking +- View counts for popularity +- Complete workflow JSON +- Creation/update timestamps +- 6-month freshness constraint + +### API Endpoints Used + +- `/api/templates/workflows` - List all workflows +- `/api/templates/search` - Search with pagination +- `/api/templates/workflows/{id}` - Get specific workflow +- `/api/templates/search/filters` - Available filters + +## Notes + +- Templates are NOT fetched during regular database rebuilds +- Run `fetch:templates` manually when you need fresh templates +- API rate limiting is implemented (200-500ms between requests) +- Progress is shown during fetching for large datasets \ No newline at end of file diff --git a/src/templates/template-fetcher.ts b/src/templates/template-fetcher.ts new file mode 100644 index 0000000..93f5b5d --- /dev/null +++ b/src/templates/template-fetcher.ts @@ -0,0 +1,149 @@ +import axios from 'axios'; +import { logger } from '../utils/logger'; + +export interface TemplateNode { + id: number; + name: string; + icon: string; +} + +export interface TemplateUser { + id: number; + name: string; + username: string; + verified: boolean; +} + +export interface TemplateWorkflow { + id: number; + name: string; + description: string; + totalViews: number; + createdAt: string; + user: TemplateUser; + nodes: TemplateNode[]; +} + +export interface TemplateDetail { + id: number; + name: string; + description: string; + views: number; + createdAt: string; + workflow: { + nodes: any[]; + connections: any; + settings?: any; + }; +} + +export class TemplateFetcher { + private readonly baseUrl = 'https://api.n8n.io/api/templates'; + private readonly pageSize = 100; + + async fetchTemplates(progressCallback?: (current: number, total: number) => void): Promise { + const sixMonthsAgo = new Date(); + sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); + + const allTemplates: TemplateWorkflow[] = []; + let page = 1; + let hasMore = true; + + logger.info('Starting template fetch from n8n.io API'); + + while (hasMore) { + try { + const response = await axios.get(`${this.baseUrl}/search`, { + params: { + page, + rows: this.pageSize, + sort_by: 'last-updated' + } + }); + + const { workflows, totalWorkflows } = response.data; + + // Filter templates by date + const recentTemplates = workflows.filter((w: TemplateWorkflow) => { + const createdDate = new Date(w.createdAt); + return createdDate >= sixMonthsAgo; + }); + + // If we hit templates older than 6 months, stop fetching + if (recentTemplates.length < workflows.length) { + hasMore = false; + logger.info(`Reached templates older than 6 months at page ${page}`); + } + + allTemplates.push(...recentTemplates); + + if (progressCallback) { + progressCallback(allTemplates.length, Math.min(totalWorkflows, allTemplates.length + 500)); + } + + // Check if there are more pages + if (workflows.length < this.pageSize || allTemplates.length >= totalWorkflows) { + hasMore = false; + } + + page++; + + // Rate limiting - be nice to the API + if (hasMore) { + await this.sleep(500); // 500ms between requests + } + } catch (error) { + logger.error(`Error fetching templates page ${page}:`, error); + throw error; + } + } + + logger.info(`Fetched ${allTemplates.length} templates from last 6 months`); + return allTemplates; + } + + async fetchTemplateDetail(workflowId: number): Promise { + try { + const response = await axios.get(`${this.baseUrl}/workflows/${workflowId}`); + return response.data.workflow; + } catch (error) { + logger.error(`Error fetching template detail for ${workflowId}:`, error); + throw error; + } + } + + async fetchAllTemplateDetails( + workflows: TemplateWorkflow[], + progressCallback?: (current: number, total: number) => void + ): Promise> { + const details = new Map(); + + logger.info(`Fetching details for ${workflows.length} templates`); + + for (let i = 0; i < workflows.length; i++) { + const workflow = workflows[i]; + + try { + const detail = await this.fetchTemplateDetail(workflow.id); + details.set(workflow.id, detail); + + if (progressCallback) { + progressCallback(i + 1, workflows.length); + } + + // Rate limiting + await this.sleep(200); // 200ms between requests + } catch (error) { + logger.error(`Failed to fetch details for workflow ${workflow.id}:`, error); + // Continue with other templates + } + } + + logger.info(`Successfully fetched ${details.size} template details`); + return details; + } + + private sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} \ No newline at end of file diff --git a/src/templates/template-repository.ts b/src/templates/template-repository.ts new file mode 100644 index 0000000..840aeca --- /dev/null +++ b/src/templates/template-repository.ts @@ -0,0 +1,181 @@ +import { DatabaseAdapter } from '../database/database-adapter'; +import { TemplateWorkflow, TemplateDetail } from './template-fetcher'; +import { logger } from '../utils/logger'; + +export interface StoredTemplate { + id: number; + workflow_id: number; + name: string; + description: string; + author_name: string; + author_username: string; + author_verified: number; + nodes_used: string; // JSON string + workflow_json: string; // JSON string + categories: string; // JSON string + views: number; + created_at: string; + updated_at: string; + url: string; + scraped_at: string; +} + +export class TemplateRepository { + constructor(private db: DatabaseAdapter) {} + + /** + * Save a template to the database + */ + saveTemplate(workflow: TemplateWorkflow, detail: TemplateDetail, categories: string[] = []): void { + const stmt = this.db.prepare(` + INSERT OR REPLACE INTO templates ( + id, workflow_id, name, description, author_name, author_username, + author_verified, nodes_used, workflow_json, categories, views, + created_at, updated_at, url + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + // Extract node types from workflow + const nodeTypes = workflow.nodes.map(n => n.name); + + // Build URL + const url = `https://n8n.io/workflows/${workflow.id}`; + + stmt.run( + workflow.id, + workflow.id, + workflow.name, + workflow.description || '', + workflow.user.name, + workflow.user.username, + workflow.user.verified ? 1 : 0, + JSON.stringify(nodeTypes), + JSON.stringify(detail.workflow), + JSON.stringify(categories), + workflow.totalViews || 0, + workflow.createdAt, + workflow.createdAt, // Using createdAt as updatedAt since API doesn't provide updatedAt + url + ); + } + + /** + * Get templates that use specific node types + */ + getTemplatesByNodes(nodeTypes: string[], limit: number = 10): StoredTemplate[] { + // Build query for multiple node types + const conditions = nodeTypes.map(() => "nodes_used LIKE ?").join(" OR "); + const query = ` + SELECT * FROM templates + WHERE ${conditions} + ORDER BY views DESC, created_at DESC + LIMIT ? + `; + + const params = [...nodeTypes.map(n => `%"${n}"%`), limit]; + return this.db.prepare(query).all(...params) as StoredTemplate[]; + } + + /** + * Get a specific template by ID + */ + getTemplate(templateId: number): StoredTemplate | null { + const row = this.db.prepare(` + SELECT * FROM templates WHERE id = ? + `).get(templateId) as StoredTemplate | undefined; + + return row || null; + } + + /** + * Search templates by name or description + */ + searchTemplates(query: string, limit: number = 20): StoredTemplate[] { + // Use FTS for search + const ftsQuery = query.split(' ').map(term => `"${term}"`).join(' OR '); + + return this.db.prepare(` + SELECT t.* FROM templates t + JOIN templates_fts ON t.id = templates_fts.rowid + WHERE templates_fts MATCH ? + ORDER BY rank, t.views DESC + LIMIT ? + `).all(ftsQuery, limit) as StoredTemplate[]; + } + + /** + * Get templates for a specific task/use case + */ + getTemplatesForTask(task: string): StoredTemplate[] { + // Map tasks to relevant node combinations + const taskNodeMap: Record = { + 'ai_automation': ['@n8n/n8n-nodes-langchain.openAi', '@n8n/n8n-nodes-langchain.agent', 'n8n-nodes-base.openAi'], + 'data_sync': ['n8n-nodes-base.googleSheets', 'n8n-nodes-base.postgres', 'n8n-nodes-base.mysql'], + 'webhook_processing': ['n8n-nodes-base.webhook', 'n8n-nodes-base.httpRequest'], + 'email_automation': ['n8n-nodes-base.gmail', 'n8n-nodes-base.emailSend', 'n8n-nodes-base.emailReadImap'], + 'slack_integration': ['n8n-nodes-base.slack', 'n8n-nodes-base.slackTrigger'], + 'data_transformation': ['n8n-nodes-base.code', 'n8n-nodes-base.set', 'n8n-nodes-base.merge'], + 'file_processing': ['n8n-nodes-base.readBinaryFile', 'n8n-nodes-base.writeBinaryFile', 'n8n-nodes-base.googleDrive'], + 'scheduling': ['n8n-nodes-base.scheduleTrigger', 'n8n-nodes-base.cron'], + 'api_integration': ['n8n-nodes-base.httpRequest', 'n8n-nodes-base.graphql'], + 'database_operations': ['n8n-nodes-base.postgres', 'n8n-nodes-base.mysql', 'n8n-nodes-base.mongodb'] + }; + + const nodes = taskNodeMap[task]; + if (!nodes) { + return []; + } + + return this.getTemplatesByNodes(nodes, 10); + } + + /** + * Get total template count + */ + getTemplateCount(): number { + const result = this.db.prepare('SELECT COUNT(*) as count FROM templates').get() as { count: number }; + return result.count; + } + + /** + * Get template statistics + */ + getTemplateStats(): Record { + const count = this.getTemplateCount(); + const avgViews = this.db.prepare('SELECT AVG(views) as avg FROM templates').get() as { avg: number }; + const topNodes = this.db.prepare(` + SELECT nodes_used FROM templates + ORDER BY views DESC + LIMIT 100 + `).all() as { nodes_used: string }[]; + + // Count node usage + const nodeCount: Record = {}; + topNodes.forEach(t => { + const nodes = JSON.parse(t.nodes_used); + nodes.forEach((n: string) => { + nodeCount[n] = (nodeCount[n] || 0) + 1; + }); + }); + + // Get top 10 most used nodes + const topUsedNodes = Object.entries(nodeCount) + .sort(([, a], [, b]) => b - a) + .slice(0, 10) + .map(([node, count]) => ({ node, count })); + + return { + totalTemplates: count, + averageViews: Math.round(avgViews.avg || 0), + topUsedNodes + }; + } + + /** + * Clear all templates (for testing or refresh) + */ + clearTemplates(): void { + this.db.exec('DELETE FROM templates'); + logger.info('Cleared all templates from database'); + } +} \ No newline at end of file diff --git a/src/templates/template-service.ts b/src/templates/template-service.ts new file mode 100644 index 0000000..3c52610 --- /dev/null +++ b/src/templates/template-service.ts @@ -0,0 +1,160 @@ +import { DatabaseAdapter } from '../database/database-adapter'; +import { TemplateRepository, StoredTemplate } from './template-repository'; +import { TemplateFetcher } from './template-fetcher'; +import { logger } from '../utils/logger'; + +export interface TemplateInfo { + id: number; + name: string; + description: string; + author: { + name: string; + username: string; + verified: boolean; + }; + nodes: string[]; + views: number; + created: string; + url: string; +} + +export interface TemplateWithWorkflow extends TemplateInfo { + workflow: any; +} + +export class TemplateService { + private repository: TemplateRepository; + private fetcher: TemplateFetcher; + + constructor(db: DatabaseAdapter) { + this.repository = new TemplateRepository(db); + this.fetcher = new TemplateFetcher(); + } + + /** + * List templates that use specific node types + */ + async listNodeTemplates(nodeTypes: string[], limit: number = 10): Promise { + const templates = this.repository.getTemplatesByNodes(nodeTypes, limit); + return templates.map(this.formatTemplateInfo); + } + + /** + * Get a specific template with full workflow + */ + async getTemplate(templateId: number): Promise { + const template = this.repository.getTemplate(templateId); + if (!template) { + return null; + } + + return { + ...this.formatTemplateInfo(template), + workflow: JSON.parse(template.workflow_json) + }; + } + + /** + * Search templates by query + */ + async searchTemplates(query: string, limit: number = 20): Promise { + const templates = this.repository.searchTemplates(query, limit); + return templates.map(this.formatTemplateInfo); + } + + /** + * Get templates for a specific task + */ + async getTemplatesForTask(task: string): Promise { + const templates = this.repository.getTemplatesForTask(task); + return templates.map(this.formatTemplateInfo); + } + + /** + * List available tasks + */ + listAvailableTasks(): string[] { + return [ + 'ai_automation', + 'data_sync', + 'webhook_processing', + 'email_automation', + 'slack_integration', + 'data_transformation', + 'file_processing', + 'scheduling', + 'api_integration', + 'database_operations' + ]; + } + + /** + * Get template statistics + */ + async getTemplateStats(): Promise> { + return this.repository.getTemplateStats(); + } + + /** + * Fetch and update templates from n8n.io + */ + async fetchAndUpdateTemplates( + progressCallback?: (message: string, current: number, total: number) => void + ): Promise { + try { + // Clear existing templates + this.repository.clearTemplates(); + + // Fetch template list + logger.info('Fetching template list from n8n.io'); + const templates = await this.fetcher.fetchTemplates((current, total) => { + progressCallback?.('Fetching template list', current, total); + }); + + logger.info(`Found ${templates.length} templates from last 6 months`); + + // Fetch details for each template + logger.info('Fetching template details'); + const details = await this.fetcher.fetchAllTemplateDetails(templates, (current, total) => { + progressCallback?.('Fetching template details', current, total); + }); + + // Save to database + logger.info('Saving templates to database'); + let saved = 0; + for (const template of templates) { + const detail = details.get(template.id); + if (detail) { + this.repository.saveTemplate(template, detail); + saved++; + } + } + + logger.info(`Successfully saved ${saved} templates to database`); + progressCallback?.('Complete', saved, saved); + } catch (error) { + logger.error('Error fetching templates:', error); + throw error; + } + } + + /** + * Format stored template for API response + */ + private formatTemplateInfo(template: StoredTemplate): TemplateInfo { + return { + id: template.id, + name: template.name, + description: template.description, + author: { + name: template.author_name, + username: template.author_username, + verified: template.author_verified === 1 + }, + nodes: JSON.parse(template.nodes_used), + views: template.views, + created: template.created_at, + url: template.url + }; + } +} \ No newline at end of file