mirror of
https://github.com/czlonkowski/n8n-mcp.git
synced 2026-03-30 14:13:12 +00:00
Compare commits
1 Commits
v2.41.4
...
feat/searc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bda2009b77 |
18
CHANGELOG.md
18
CHANGELOG.md
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.42.0] - 2026-03-30
|
||||
|
||||
### Added
|
||||
|
||||
- **`includeOperations` flag for search_nodes**: Opt-in parameter that returns a resource/operation tree per search result, grouped by resource (e.g., Slack returns 7 resources with 44 operations). Saves a mandatory `get_node` round-trip when building workflows. Adds ~100-300 tokens per result.
|
||||
|
||||
- **`searchMode: "patterns"` for search_templates**: New lightweight mode that serves workflow pattern summaries mined from 2,700+ templates. Returns common node combinations, connection chains, and frequency data per task category (10 categories: ai_automation, webhook_processing, scheduling, etc.). Use `task` parameter for category-specific patterns or omit for overview.
|
||||
|
||||
- **Workflow pattern mining script** (`npm run mine:patterns`): Extracts node frequency, co-occurrence, and connection topology from the template database. Two-pass pipeline: Pass 1 analyzes `nodes_used` metadata (no decompression), Pass 2 decompresses workflows for connection analysis. Produces `data/workflow-patterns.json` with 554 node types, 3,201 edges, and 5,246 chains.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Operations extraction now includes resource grouping**: The property extractor was using `find()` to get only the first `operation` property, but n8n nodes have multiple operation properties each mapped to a different resource via `displayOptions.show.resource`. Changed to `filter()` to capture all operation properties. Slack went from 17 flat operations to 44 operations across 7 named resources.
|
||||
|
||||
- **FTS-to-LIKE fallback dropped search options**: When the FTS5 search fell back to LIKE-based search (e.g., for "http request"), the `options` object (including `includeOperations`, `includeExamples`, `source`) was silently lost. Now correctly passed through.
|
||||
|
||||
Conceived by Romuald Członkowski - https://www.aiadvisors.pl/en
|
||||
|
||||
## [2.41.4] - 2026-03-30
|
||||
|
||||
### Fixed
|
||||
|
||||
BIN
data/nodes.db
BIN
data/nodes.db
Binary file not shown.
2670
data/workflow-patterns.json
Normal file
2670
data/workflow-patterns.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "n8n-mcp",
|
||||
"version": "2.41.4",
|
||||
"version": "2.42.0",
|
||||
"description": "Integration between n8n workflow automation and Model Context Protocol (MCP)",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
@@ -90,6 +90,7 @@
|
||||
"test:docker:unit": "./scripts/test-docker-config.sh unit",
|
||||
"test:docker:integration": "./scripts/test-docker-config.sh integration",
|
||||
"test:docker:security": "./scripts/test-docker-config.sh security",
|
||||
"mine:patterns": "tsx src/scripts/mine-workflow-patterns.ts",
|
||||
"sanitize:templates": "node dist/scripts/sanitize-templates.js",
|
||||
"db:rebuild": "node dist/scripts/rebuild-database.js",
|
||||
"db:init": "node -e \"new (require('./dist/services/sqlite-storage-service').SQLiteStorageService)(); console.log('Database initialized')\"",
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
ListResourcesRequestSchema,
|
||||
ReadResourceRequestSchema,
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
import { existsSync, promises as fs } from 'fs';
|
||||
import { existsSync, readFileSync, promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { n8nDocumentationToolsFinal } from './tools';
|
||||
import { UIAppRegistry } from './ui';
|
||||
@@ -1310,6 +1310,7 @@ export class N8NDocumentationMCPServer {
|
||||
return this.searchNodes(args.query, limit, {
|
||||
mode: args.mode,
|
||||
includeExamples: args.includeExamples,
|
||||
includeOperations: args.includeOperations,
|
||||
source: args.source
|
||||
});
|
||||
case 'get_node':
|
||||
@@ -1414,6 +1415,8 @@ export class N8NDocumentationMCPServer {
|
||||
requiredService: args.requiredService,
|
||||
targetAudience: args.targetAudience
|
||||
}, searchLimit, searchOffset);
|
||||
case 'patterns':
|
||||
return this.getWorkflowPatterns(args.task as string | undefined, searchLimit);
|
||||
case 'keyword':
|
||||
default:
|
||||
if (!args.query) {
|
||||
@@ -1681,6 +1684,7 @@ export class N8NDocumentationMCPServer {
|
||||
mode?: 'OR' | 'AND' | 'FUZZY';
|
||||
includeSource?: boolean;
|
||||
includeExamples?: boolean;
|
||||
includeOperations?: boolean;
|
||||
source?: 'all' | 'core' | 'community' | 'verified';
|
||||
}
|
||||
): Promise<any> {
|
||||
@@ -1723,6 +1727,7 @@ export class N8NDocumentationMCPServer {
|
||||
options?: {
|
||||
includeSource?: boolean;
|
||||
includeExamples?: boolean;
|
||||
includeOperations?: boolean;
|
||||
source?: 'all' | 'core' | 'community' | 'verified';
|
||||
}
|
||||
): Promise<any> {
|
||||
@@ -1736,7 +1741,7 @@ export class N8NDocumentationMCPServer {
|
||||
|
||||
// For FUZZY mode, use LIKE search with typo patterns
|
||||
if (mode === 'FUZZY') {
|
||||
return this.searchNodesFuzzy(cleanedQuery, limit);
|
||||
return this.searchNodesFuzzy(cleanedQuery, limit, { includeOperations: options?.includeOperations });
|
||||
}
|
||||
|
||||
let ftsQuery: string;
|
||||
@@ -1827,7 +1832,7 @@ export class N8NDocumentationMCPServer {
|
||||
if (cleanedQuery.toLowerCase().includes('http') && !hasHttpRequest) {
|
||||
// FTS missed HTTP Request, fall back to LIKE search
|
||||
logger.debug('FTS missed HTTP Request node, augmenting with LIKE search');
|
||||
return this.searchNodesLIKE(query, limit);
|
||||
return this.searchNodesLIKE(query, limit, options);
|
||||
}
|
||||
|
||||
const result: any = {
|
||||
@@ -1855,6 +1860,14 @@ export class N8NDocumentationMCPServer {
|
||||
}
|
||||
}
|
||||
|
||||
// Add operations tree if requested
|
||||
if (options?.includeOperations) {
|
||||
const opsTree = this.buildOperationsTree(node.operations);
|
||||
if (opsTree) {
|
||||
nodeResult.operationsTree = opsTree;
|
||||
}
|
||||
}
|
||||
|
||||
return nodeResult;
|
||||
}),
|
||||
totalCount: scoredNodes.length
|
||||
@@ -1922,7 +1935,13 @@ export class N8NDocumentationMCPServer {
|
||||
}
|
||||
}
|
||||
|
||||
private async searchNodesFuzzy(query: string, limit: number): Promise<any> {
|
||||
private async searchNodesFuzzy(
|
||||
query: string,
|
||||
limit: number,
|
||||
options?: {
|
||||
includeOperations?: boolean;
|
||||
}
|
||||
): Promise<any> {
|
||||
if (!this.db) throw new Error('Database not initialized');
|
||||
|
||||
// Split into words for fuzzy matching
|
||||
@@ -1963,14 +1982,26 @@ export class N8NDocumentationMCPServer {
|
||||
return {
|
||||
query,
|
||||
mode: 'FUZZY',
|
||||
results: matchingNodes.map(node => ({
|
||||
nodeType: node.node_type,
|
||||
workflowNodeType: getWorkflowNodeType(node.package_name, node.node_type),
|
||||
displayName: node.display_name,
|
||||
description: node.description,
|
||||
category: node.category,
|
||||
package: node.package_name
|
||||
})),
|
||||
results: matchingNodes.map(node => {
|
||||
const nodeResult: any = {
|
||||
nodeType: node.node_type,
|
||||
workflowNodeType: getWorkflowNodeType(node.package_name, node.node_type),
|
||||
displayName: node.display_name,
|
||||
description: node.description,
|
||||
category: node.category,
|
||||
package: node.package_name
|
||||
};
|
||||
|
||||
// Add operations tree if requested
|
||||
if (options?.includeOperations) {
|
||||
const opsTree = this.buildOperationsTree(node.operations);
|
||||
if (opsTree) {
|
||||
nodeResult.operationsTree = opsTree;
|
||||
}
|
||||
}
|
||||
|
||||
return nodeResult;
|
||||
}),
|
||||
totalCount: matchingNodes.length
|
||||
};
|
||||
}
|
||||
@@ -2075,6 +2106,7 @@ export class N8NDocumentationMCPServer {
|
||||
options?: {
|
||||
includeSource?: boolean;
|
||||
includeExamples?: boolean;
|
||||
includeOperations?: boolean;
|
||||
source?: 'all' | 'core' | 'community' | 'verified';
|
||||
}
|
||||
): Promise<any> {
|
||||
@@ -2134,6 +2166,14 @@ export class N8NDocumentationMCPServer {
|
||||
}
|
||||
}
|
||||
|
||||
// Add operations tree if requested
|
||||
if (options?.includeOperations) {
|
||||
const opsTree = this.buildOperationsTree(node.operations);
|
||||
if (opsTree) {
|
||||
nodeResult.operationsTree = opsTree;
|
||||
}
|
||||
}
|
||||
|
||||
return nodeResult;
|
||||
}),
|
||||
totalCount: rankedNodes.length
|
||||
@@ -2220,6 +2260,14 @@ export class N8NDocumentationMCPServer {
|
||||
}
|
||||
}
|
||||
|
||||
// Add operations tree if requested
|
||||
if (options?.includeOperations) {
|
||||
const opsTree = this.buildOperationsTree(node.operations);
|
||||
if (opsTree) {
|
||||
nodeResult.operationsTree = opsTree;
|
||||
}
|
||||
}
|
||||
|
||||
return nodeResult;
|
||||
}),
|
||||
totalCount: rankedNodes.length
|
||||
@@ -2590,6 +2638,51 @@ Full documentation is being prepared. For now, use get_node_essentials for confi
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse raw operations data and group by resource into a compact tree.
|
||||
* Returns undefined when there are no operations (e.g. trigger nodes, Code node).
|
||||
*/
|
||||
private buildOperationsTree(operationsRaw: string | any[] | null | undefined): Array<{resource: string, operations: string[]}> | undefined {
|
||||
if (!operationsRaw) return undefined;
|
||||
|
||||
let ops: any[];
|
||||
if (typeof operationsRaw === 'string') {
|
||||
try {
|
||||
ops = JSON.parse(operationsRaw);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
} else if (Array.isArray(operationsRaw)) {
|
||||
ops = operationsRaw;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!Array.isArray(ops) || ops.length === 0) return undefined;
|
||||
|
||||
// Group by resource
|
||||
const byResource = new Map<string, string[]>();
|
||||
for (const op of ops) {
|
||||
const resource = op.resource || 'default';
|
||||
const opName = op.name || op.operation;
|
||||
if (!opName) continue;
|
||||
if (!byResource.has(resource)) {
|
||||
byResource.set(resource, []);
|
||||
}
|
||||
const list = byResource.get(resource)!;
|
||||
if (!list.includes(opName)) {
|
||||
list.push(opName);
|
||||
}
|
||||
}
|
||||
|
||||
if (byResource.size === 0) return undefined;
|
||||
|
||||
return Array.from(byResource.entries()).map(([resource, operations]) => ({
|
||||
resource,
|
||||
operations
|
||||
}));
|
||||
}
|
||||
|
||||
private async getNodeEssentials(nodeType: string, includeExamples?: boolean): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.repository) throw new Error('Repository not initialized');
|
||||
@@ -3787,6 +3880,65 @@ Full documentation is being prepared. For now, use get_node_essentials for confi
|
||||
};
|
||||
}
|
||||
|
||||
private workflowPatternsCache: {
|
||||
generatedAt: string;
|
||||
templateCount: number;
|
||||
categories: Record<string, {
|
||||
templateCount: number;
|
||||
pattern: string;
|
||||
nodes?: Array<{ type: string; frequency: number; role: string; displayName: string }>;
|
||||
commonChains?: Array<{ chain: string[]; count: number; frequency: number }>;
|
||||
}>;
|
||||
} | null = null;
|
||||
|
||||
private getWorkflowPatterns(category?: string, limit: number = 10): any {
|
||||
// Load patterns file (cached after first load)
|
||||
if (!this.workflowPatternsCache) {
|
||||
try {
|
||||
const patternsPath = path.join(__dirname, '..', '..', 'data', 'workflow-patterns.json');
|
||||
if (existsSync(patternsPath)) {
|
||||
this.workflowPatternsCache = JSON.parse(readFileSync(patternsPath, 'utf-8'));
|
||||
} else {
|
||||
return { error: 'Workflow patterns not generated yet. Run: npm run mine:patterns' };
|
||||
}
|
||||
} catch (e) {
|
||||
return { error: `Failed to load workflow patterns: ${e instanceof Error ? e.message : String(e)}` };
|
||||
}
|
||||
}
|
||||
|
||||
const patterns = this.workflowPatternsCache!;
|
||||
|
||||
if (category) {
|
||||
// Return specific category pattern data
|
||||
const categoryData = patterns.categories[category];
|
||||
if (!categoryData) {
|
||||
const available = Object.keys(patterns.categories);
|
||||
return { error: `Unknown category "${category}". Available: ${available.join(', ')}` };
|
||||
}
|
||||
return {
|
||||
category,
|
||||
...categoryData,
|
||||
nodes: categoryData.nodes?.slice(0, limit),
|
||||
commonChains: categoryData.commonChains?.slice(0, limit),
|
||||
};
|
||||
}
|
||||
|
||||
// Return overview of all categories
|
||||
const overview = Object.entries(patterns.categories).map(([name, data]) => ({
|
||||
category: name,
|
||||
templateCount: data.templateCount,
|
||||
pattern: data.pattern,
|
||||
topNodes: data.nodes?.slice(0, 5).map(n => n.displayName || n.type),
|
||||
}));
|
||||
|
||||
return {
|
||||
templateCount: patterns.templateCount,
|
||||
generatedAt: patterns.generatedAt,
|
||||
categories: overview,
|
||||
tip: 'Use search_templates({searchMode: "patterns", task: "category_name"}) for full pattern data with nodes, chains, and tips.',
|
||||
};
|
||||
}
|
||||
|
||||
private async getTemplatesForTask(task: string, limit: number = 10, offset: number = 0): Promise<any> {
|
||||
await this.ensureInitialized();
|
||||
if (!this.templateService) throw new Error('Template service not initialized');
|
||||
|
||||
@@ -5,7 +5,7 @@ export const searchNodesDoc: ToolDocumentation = {
|
||||
category: 'discovery',
|
||||
essentials: {
|
||||
description: 'Text search across node names and descriptions. Returns most relevant nodes first, with frequently-used nodes (HTTP Request, Webhook, Set, Code, Slack) prioritized in results. Searches all 800+ nodes including 300+ verified community nodes.',
|
||||
keyParameters: ['query', 'mode', 'limit', 'source', 'includeExamples'],
|
||||
keyParameters: ['query', 'mode', 'limit', 'source', 'includeExamples', 'includeOperations'],
|
||||
example: 'search_nodes({query: "webhook"})',
|
||||
performance: '<20ms even for complex queries',
|
||||
tips: [
|
||||
@@ -14,7 +14,8 @@ export const searchNodesDoc: ToolDocumentation = {
|
||||
'FUZZY mode: Handles typos and spelling errors',
|
||||
'Use quotes for exact phrases: "google sheets"',
|
||||
'Use source="community" to search only community nodes',
|
||||
'Use source="verified" for verified community nodes only'
|
||||
'Use source="verified" for verified community nodes only',
|
||||
'Use includeOperations=true to get resource/operation trees without a separate get_node call'
|
||||
]
|
||||
},
|
||||
full: {
|
||||
@@ -24,7 +25,8 @@ export const searchNodesDoc: ToolDocumentation = {
|
||||
limit: { type: 'number', description: 'Maximum results to return. Default: 20, Max: 100', required: false },
|
||||
mode: { type: 'string', description: 'Search mode: "OR" (any word matches, default), "AND" (all words required), "FUZZY" (typo-tolerant)', required: false },
|
||||
source: { type: 'string', description: 'Filter by node source: "all" (default, everything), "core" (n8n base nodes only), "community" (community nodes only), "verified" (verified community nodes only)', required: false },
|
||||
includeExamples: { type: 'boolean', description: 'Include top 2 real-world configuration examples from popular templates for each node. Default: false. Adds ~200-400 tokens per node.', required: false }
|
||||
includeExamples: { type: 'boolean', description: 'Include top 2 real-world configuration examples from popular templates for each node. Default: false. Adds ~200-400 tokens per node.', required: false },
|
||||
includeOperations: { type: 'boolean', description: 'Include resource/operation tree per node. Default: false. Adds ~100-300 tokens per result but saves a get_node round-trip.', required: false }
|
||||
},
|
||||
returns: 'Array of node objects sorted by relevance score. Each object contains: nodeType, displayName, description, category, relevance score. For community nodes, also includes: isCommunity (boolean), isVerified (boolean), authorName (string), npmDownloads (number). Common nodes appear first when relevance is similar.',
|
||||
examples: [
|
||||
@@ -37,7 +39,8 @@ export const searchNodesDoc: ToolDocumentation = {
|
||||
'search_nodes({query: "scraping", source: "community"}) - Find community scraping nodes',
|
||||
'search_nodes({query: "pdf", source: "verified"}) - Find verified community PDF nodes',
|
||||
'search_nodes({query: "brightdata"}) - Find BrightData community node',
|
||||
'search_nodes({query: "slack", includeExamples: true}) - Get Slack with template examples'
|
||||
'search_nodes({query: "slack", includeExamples: true}) - Get Slack with template examples',
|
||||
'search_nodes({query: "slack", includeOperations: true}) - Get Slack with resource/operation tree'
|
||||
],
|
||||
useCases: [
|
||||
'Finding nodes when you know partial names',
|
||||
|
||||
@@ -57,6 +57,11 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
|
||||
description: 'Include top 2 real-world configuration examples from popular templates (default: false)',
|
||||
default: false,
|
||||
},
|
||||
includeOperations: {
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Include resource/operation tree per node. Adds ~100-300 tokens per result but saves a get_node round-trip.',
|
||||
},
|
||||
source: {
|
||||
type: 'string',
|
||||
enum: ['all', 'core', 'community', 'verified'],
|
||||
@@ -242,14 +247,14 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
|
||||
},
|
||||
{
|
||||
name: 'search_templates',
|
||||
description: `Search templates with multiple modes. Use searchMode='keyword' for text search, 'by_nodes' to find templates using specific nodes, 'by_task' for curated task-based templates, 'by_metadata' for filtering by complexity/setup time/services.`,
|
||||
description: `Search templates with multiple modes. Use searchMode='keyword' for text search, 'by_nodes' to find templates using specific nodes, 'by_task' for curated task-based templates, 'by_metadata' for filtering by complexity/setup time/services, 'patterns' for lightweight workflow pattern summaries mined from 2700+ templates.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
searchMode: {
|
||||
type: 'string',
|
||||
enum: ['keyword', 'by_nodes', 'by_task', 'by_metadata'],
|
||||
description: 'Search mode. keyword=text search (default), by_nodes=find by node types, by_task=curated task templates, by_metadata=filter by complexity/services',
|
||||
enum: ['keyword', 'by_nodes', 'by_task', 'by_metadata', 'patterns'],
|
||||
description: 'Search mode. keyword=text search (default), by_nodes=find by node types, by_task=curated task templates, by_metadata=filter by complexity/services, patterns=lightweight workflow pattern summaries',
|
||||
default: 'keyword',
|
||||
},
|
||||
// For searchMode='keyword'
|
||||
@@ -271,7 +276,7 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
|
||||
items: { type: 'string' },
|
||||
description: 'For searchMode=by_nodes: array of node types (e.g., ["n8n-nodes-base.httpRequest", "n8n-nodes-base.slack"])',
|
||||
},
|
||||
// For searchMode='by_task'
|
||||
// For searchMode='by_task' or 'patterns'
|
||||
task: {
|
||||
type: 'string',
|
||||
enum: [
|
||||
@@ -286,7 +291,7 @@ export const n8nDocumentationToolsFinal: ToolDefinition[] = [
|
||||
'api_integration',
|
||||
'database_operations'
|
||||
],
|
||||
description: 'For searchMode=by_task: the type of task',
|
||||
description: 'For searchMode=by_task: the type of task. For searchMode=patterns: optional category filter (omit for overview of all categories).',
|
||||
},
|
||||
// For searchMode='by_metadata'
|
||||
category: {
|
||||
|
||||
@@ -131,18 +131,23 @@ export class PropertyExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
// Programmatic nodes - look for operation property in properties
|
||||
// Programmatic nodes - look for operation properties in properties
|
||||
// Note: nodes can have MULTIPLE operation properties, each with displayOptions.show.resource
|
||||
// mapping to a different resource (e.g., Slack has 7 operation props for channel, message, etc.)
|
||||
if (description.properties && Array.isArray(description.properties)) {
|
||||
const operationProp = description.properties.find(
|
||||
const operationProps = description.properties.filter(
|
||||
(p: any) => p.name === 'operation' || p.name === 'action'
|
||||
);
|
||||
|
||||
if (operationProp?.options) {
|
||||
|
||||
for (const operationProp of operationProps) {
|
||||
if (!operationProp?.options) continue;
|
||||
const resource = operationProp.displayOptions?.show?.resource?.[0];
|
||||
operationProp.options.forEach((op: any) => {
|
||||
operations.push({
|
||||
operation: op.value,
|
||||
name: op.name,
|
||||
description: op.description
|
||||
description: op.description,
|
||||
...(resource ? { resource } : {})
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
532
src/scripts/mine-workflow-patterns.ts
Normal file
532
src/scripts/mine-workflow-patterns.ts
Normal file
@@ -0,0 +1,532 @@
|
||||
#!/usr/bin/env node
|
||||
import { createDatabaseAdapter } from '../database/database-adapter';
|
||||
import * as zlib from 'zlib';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// Node types to exclude from analysis
|
||||
const EXCLUDED_TYPES = new Set([
|
||||
'n8n-nodes-base.stickyNote',
|
||||
'n8n-nodes-base.noOp',
|
||||
'n8n-nodes-base.manualTrigger',
|
||||
]);
|
||||
|
||||
// Category-to-node mapping for classification
|
||||
const TASK_NODE_MAPPING: Record<string, string[]> = {
|
||||
ai_automation: [
|
||||
'nodes-langchain.agent', 'nodes-langchain.openAi', 'nodes-langchain.chainLlm',
|
||||
'nodes-langchain.lmChatOpenAi', 'nodes-langchain.lmChatAnthropic',
|
||||
'nodes-langchain.chainSummarization', 'nodes-langchain.toolWorkflow',
|
||||
'nodes-langchain.memoryBufferWindow', 'nodes-langchain.outputParserStructured',
|
||||
],
|
||||
webhook_processing: [
|
||||
'nodes-base.webhook', 'nodes-base.respondToWebhook',
|
||||
],
|
||||
email_automation: [
|
||||
'nodes-base.gmail', 'nodes-base.emailSend', 'nodes-base.microsoftOutlook',
|
||||
'nodes-base.emailReadImap',
|
||||
],
|
||||
slack_integration: [
|
||||
'nodes-base.slack', 'nodes-base.slackTrigger',
|
||||
],
|
||||
data_sync: [
|
||||
'nodes-base.googleSheets', 'nodes-base.airtable', 'nodes-base.postgres',
|
||||
'nodes-base.mysql', 'nodes-base.mongoDb',
|
||||
],
|
||||
data_transformation: [
|
||||
'nodes-base.set', 'nodes-base.code', 'nodes-base.splitInBatches',
|
||||
'nodes-base.merge', 'nodes-base.itemLists', 'nodes-base.filter',
|
||||
'nodes-base.if', 'nodes-base.switch',
|
||||
],
|
||||
scheduling: [
|
||||
'nodes-base.scheduleTrigger', 'nodes-base.cron',
|
||||
],
|
||||
api_integration: [
|
||||
'nodes-base.httpRequest', 'nodes-base.webhook', 'nodes-base.graphql',
|
||||
],
|
||||
database_operations: [
|
||||
'nodes-base.postgres', 'nodes-base.mongoDb', 'nodes-base.redis',
|
||||
'nodes-base.mysql', 'nodes-base.mySql',
|
||||
],
|
||||
file_processing: [
|
||||
'nodes-base.readBinaryFiles', 'nodes-base.writeBinaryFile',
|
||||
'nodes-base.spreadsheetFile', 'nodes-base.googleDrive',
|
||||
],
|
||||
};
|
||||
|
||||
// Display name mapping for common node types (used for pattern strings)
|
||||
const DISPLAY_NAMES: Record<string, string> = {
|
||||
'n8n-nodes-base.webhook': 'Webhook',
|
||||
'n8n-nodes-base.httpRequest': 'HTTP Request',
|
||||
'n8n-nodes-base.code': 'Code',
|
||||
'n8n-nodes-base.set': 'Set',
|
||||
'n8n-nodes-base.if': 'If',
|
||||
'n8n-nodes-base.switch': 'Switch',
|
||||
'n8n-nodes-base.merge': 'Merge',
|
||||
'n8n-nodes-base.filter': 'Filter',
|
||||
'n8n-nodes-base.splitInBatches': 'Split In Batches',
|
||||
'n8n-nodes-base.itemLists': 'Item Lists',
|
||||
'n8n-nodes-base.respondToWebhook': 'Respond to Webhook',
|
||||
'n8n-nodes-base.gmail': 'Gmail',
|
||||
'n8n-nodes-base.emailSend': 'Send Email',
|
||||
'n8n-nodes-base.slack': 'Slack',
|
||||
'n8n-nodes-base.slackTrigger': 'Slack Trigger',
|
||||
'n8n-nodes-base.googleSheets': 'Google Sheets',
|
||||
'n8n-nodes-base.airtable': 'Airtable',
|
||||
'n8n-nodes-base.postgres': 'Postgres',
|
||||
'n8n-nodes-base.mysql': 'MySQL',
|
||||
'n8n-nodes-base.mongoDb': 'MongoDB',
|
||||
'n8n-nodes-base.redis': 'Redis',
|
||||
'n8n-nodes-base.scheduleTrigger': 'Schedule Trigger',
|
||||
'n8n-nodes-base.cron': 'Cron',
|
||||
'n8n-nodes-base.googleDrive': 'Google Drive',
|
||||
'n8n-nodes-base.spreadsheetFile': 'Spreadsheet File',
|
||||
'n8n-nodes-base.readBinaryFiles': 'Read Binary Files',
|
||||
'n8n-nodes-base.writeBinaryFile': 'Write Binary File',
|
||||
'n8n-nodes-base.graphql': 'GraphQL',
|
||||
'n8n-nodes-base.microsoftOutlook': 'Microsoft Outlook',
|
||||
'n8n-nodes-base.emailReadImap': 'Email (IMAP)',
|
||||
'n8n-nodes-base.noOp': 'No Op',
|
||||
'@n8n/n8n-nodes-langchain.agent': 'AI Agent',
|
||||
'@n8n/n8n-nodes-langchain.openAi': 'OpenAI',
|
||||
'@n8n/n8n-nodes-langchain.chainLlm': 'LLM Chain',
|
||||
'@n8n/n8n-nodes-langchain.lmChatOpenAi': 'OpenAI Chat Model',
|
||||
'@n8n/n8n-nodes-langchain.lmChatAnthropic': 'Anthropic Chat Model',
|
||||
'@n8n/n8n-nodes-langchain.chainSummarization': 'Summarization Chain',
|
||||
'@n8n/n8n-nodes-langchain.toolWorkflow': 'Workflow Tool',
|
||||
'@n8n/n8n-nodes-langchain.memoryBufferWindow': 'Window Buffer Memory',
|
||||
'@n8n/n8n-nodes-langchain.outputParserStructured': 'Structured Output Parser',
|
||||
'n8n-nodes-base.manualTrigger': 'Manual Trigger',
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if a node type matches a category mapping entry.
|
||||
* Category mappings use short forms like 'nodes-base.webhook'
|
||||
* while actual types may be 'n8n-nodes-base.webhook' or '@n8n/n8n-nodes-langchain.agent'.
|
||||
*/
|
||||
function matchesCategory(nodeType: string, categoryPattern: string): boolean {
|
||||
return nodeType.endsWith(categoryPattern) || nodeType.includes(categoryPattern);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get display name for a node type.
|
||||
*/
|
||||
function getDisplayName(nodeType: string): string {
|
||||
if (DISPLAY_NAMES[nodeType]) {
|
||||
return DISPLAY_NAMES[nodeType];
|
||||
}
|
||||
// Extract the last part after the last dot
|
||||
const parts = nodeType.split('.');
|
||||
const name = parts[parts.length - 1];
|
||||
// Convert camelCase to Title Case
|
||||
return name
|
||||
.replace(/([A-Z])/g, ' $1')
|
||||
.replace(/^./, s => s.toUpperCase())
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if a node type is a trigger node.
|
||||
*/
|
||||
function isTriggerType(nodeType: string): boolean {
|
||||
const lower = nodeType.toLowerCase();
|
||||
return lower.includes('trigger') || lower.includes('webhook');
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a set of node types into categories.
|
||||
*/
|
||||
function classifyTemplate(nodeTypes: string[], metadataCategories?: string[]): string[] {
|
||||
const categories = new Set<string>();
|
||||
|
||||
for (const nodeType of nodeTypes) {
|
||||
for (const [category, patterns] of Object.entries(TASK_NODE_MAPPING)) {
|
||||
for (const pattern of patterns) {
|
||||
if (matchesCategory(nodeType, pattern)) {
|
||||
categories.add(category);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also include categories from metadata_json if available
|
||||
if (metadataCategories && Array.isArray(metadataCategories)) {
|
||||
for (const cat of metadataCategories) {
|
||||
const normalized = cat.toLowerCase().replace(/[\s-]+/g, '_');
|
||||
// Map metadata categories to our category keys
|
||||
for (const key of Object.keys(TASK_NODE_MAPPING)) {
|
||||
if (normalized.includes(key) || key.includes(normalized)) {
|
||||
categories.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(categories);
|
||||
}
|
||||
|
||||
interface NodeFrequency {
|
||||
type: string;
|
||||
count: number;
|
||||
frequency: number;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
interface EdgeFrequency {
|
||||
from: string;
|
||||
to: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface ChainFrequency {
|
||||
chain: string[];
|
||||
count: number;
|
||||
frequency: number;
|
||||
}
|
||||
|
||||
interface CategoryData {
|
||||
templateCount: number;
|
||||
pattern: string;
|
||||
nodes: Array<{ type: string; frequency: number; role: string; displayName: string }>;
|
||||
commonChains: ChainFrequency[];
|
||||
}
|
||||
|
||||
interface PatternOutput {
|
||||
generatedAt: string;
|
||||
templateCount: number;
|
||||
categories: Record<string, CategoryData>;
|
||||
global: {
|
||||
topNodes: NodeFrequency[];
|
||||
topEdges: EdgeFrequency[];
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const dbPath = path.resolve(__dirname, '../../data/nodes.db');
|
||||
console.log(`Opening database: ${dbPath}`);
|
||||
const db = await createDatabaseAdapter(dbPath);
|
||||
|
||||
// ---- Pass 1: Frequency & co-occurrence ----
|
||||
console.log('\n=== Pass 1: Node frequency & co-occurrence ===');
|
||||
const pass1Start = Date.now();
|
||||
|
||||
const lightRows = db.prepare(
|
||||
'SELECT id, nodes_used, metadata_json, views FROM templates ORDER BY views DESC'
|
||||
).all() as Array<{ id: number; nodes_used: string | null; metadata_json: string | null; views: number }>;
|
||||
|
||||
const templateCount = lightRows.length;
|
||||
console.log(`Found ${templateCount} templates`);
|
||||
|
||||
// Global counters
|
||||
const nodeFrequency = new Map<string, number>();
|
||||
const pairCooccurrence = new Map<string, number>();
|
||||
// Per-category tracking
|
||||
const categoryTemplates = new Map<string, Set<number>>();
|
||||
const categoryNodes = new Map<string, Map<string, number>>();
|
||||
|
||||
for (let i = 0; i < lightRows.length; i++) {
|
||||
const row = lightRows[i];
|
||||
if (i > 0 && i % 500 === 0) {
|
||||
console.log(` Processing template ${i}/${templateCount}...`);
|
||||
}
|
||||
|
||||
if (!row.nodes_used) continue;
|
||||
|
||||
let nodeTypes: string[];
|
||||
try {
|
||||
nodeTypes = JSON.parse(row.nodes_used);
|
||||
if (!Array.isArray(nodeTypes)) continue;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deduplicate and filter
|
||||
const uniqueTypes = [...new Set(nodeTypes)].filter(t => !EXCLUDED_TYPES.has(t));
|
||||
if (uniqueTypes.length === 0) continue;
|
||||
|
||||
// Count global frequency
|
||||
for (const nodeType of uniqueTypes) {
|
||||
nodeFrequency.set(nodeType, (nodeFrequency.get(nodeType) || 0) + 1);
|
||||
}
|
||||
|
||||
// Count pairwise co-occurrence
|
||||
for (let a = 0; a < uniqueTypes.length; a++) {
|
||||
for (let b = a + 1; b < uniqueTypes.length; b++) {
|
||||
const pair = [uniqueTypes[a], uniqueTypes[b]].sort().join('|||');
|
||||
pairCooccurrence.set(pair, (pairCooccurrence.get(pair) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Classify into categories
|
||||
let metadataCategories: string[] | undefined;
|
||||
if (row.metadata_json) {
|
||||
try {
|
||||
const meta = JSON.parse(row.metadata_json);
|
||||
metadataCategories = meta.categories;
|
||||
} catch {
|
||||
// skip
|
||||
}
|
||||
}
|
||||
|
||||
const categories = classifyTemplate(uniqueTypes, metadataCategories);
|
||||
for (const cat of categories) {
|
||||
if (!categoryTemplates.has(cat)) {
|
||||
categoryTemplates.set(cat, new Set());
|
||||
categoryNodes.set(cat, new Map());
|
||||
}
|
||||
categoryTemplates.get(cat)!.add(row.id);
|
||||
const catNodeMap = categoryNodes.get(cat)!;
|
||||
for (const nodeType of uniqueTypes) {
|
||||
catNodeMap.set(nodeType, (catNodeMap.get(nodeType) || 0) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pass1Time = ((Date.now() - pass1Start) / 1000).toFixed(1);
|
||||
console.log(`Pass 1 complete: ${pass1Time}s`);
|
||||
console.log(` Unique node types: ${nodeFrequency.size}`);
|
||||
console.log(` Categories found: ${categoryTemplates.size}`);
|
||||
|
||||
// ---- Pass 2: Connection topology ----
|
||||
console.log('\n=== Pass 2: Connection topology ===');
|
||||
const pass2Start = Date.now();
|
||||
|
||||
const compressedRows = db.prepare(
|
||||
'SELECT id, nodes_used, workflow_json_compressed, views FROM templates ORDER BY views DESC'
|
||||
).all() as Array<{ id: number; nodes_used: string | null; workflow_json_compressed: string | null; views: number }>;
|
||||
|
||||
// Edge frequency: sourceType -> targetType
|
||||
const edgeFrequency = new Map<string, number>();
|
||||
// Chain frequency (by category)
|
||||
const categoryChains = new Map<string, Map<string, number>>();
|
||||
// Global chains
|
||||
const globalChains = new Map<string, number>();
|
||||
|
||||
let decompressedCount = 0;
|
||||
let decompressFailCount = 0;
|
||||
|
||||
for (let i = 0; i < compressedRows.length; i++) {
|
||||
const row = compressedRows[i];
|
||||
if (i > 0 && i % 500 === 0) {
|
||||
console.log(` Processing template ${i}/${templateCount}...`);
|
||||
}
|
||||
|
||||
if (!row.workflow_json_compressed) continue;
|
||||
|
||||
let workflow: any;
|
||||
try {
|
||||
const decompressed = zlib.gunzipSync(Buffer.from(row.workflow_json_compressed, 'base64'));
|
||||
workflow = JSON.parse(decompressed.toString());
|
||||
decompressedCount++;
|
||||
} catch {
|
||||
decompressFailCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const nodes: any[] = workflow.nodes || [];
|
||||
const connections: Record<string, any> = workflow.connections || {};
|
||||
|
||||
// Build name -> type map
|
||||
const nameToType = new Map<string, string>();
|
||||
for (const node of nodes) {
|
||||
if (node.name && node.type && !EXCLUDED_TYPES.has(node.type)) {
|
||||
nameToType.set(node.name, node.type);
|
||||
}
|
||||
}
|
||||
|
||||
// Build adjacency list for BFS
|
||||
const adjacency = new Map<string, string[]>();
|
||||
|
||||
// Parse connections and record edges
|
||||
for (const sourceName of Object.keys(connections)) {
|
||||
const sourceType = nameToType.get(sourceName);
|
||||
if (!sourceType) continue;
|
||||
|
||||
const mainOutputs = connections[sourceName]?.main;
|
||||
if (!Array.isArray(mainOutputs)) continue;
|
||||
|
||||
for (const outputGroup of mainOutputs) {
|
||||
if (!Array.isArray(outputGroup)) continue;
|
||||
for (const conn of outputGroup) {
|
||||
if (!conn || !conn.node) continue;
|
||||
const targetName = conn.node;
|
||||
const targetType = nameToType.get(targetName);
|
||||
if (!targetType) continue;
|
||||
|
||||
// Record edge
|
||||
const edgeKey = `${sourceType}|||${targetType}`;
|
||||
edgeFrequency.set(edgeKey, (edgeFrequency.get(edgeKey) || 0) + 1);
|
||||
|
||||
// Build adjacency for chain extraction
|
||||
if (!adjacency.has(sourceName)) {
|
||||
adjacency.set(sourceName, []);
|
||||
}
|
||||
adjacency.get(sourceName)!.push(targetName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find trigger nodes (nodes with no incoming connections, or type contains 'Trigger')
|
||||
const hasIncoming = new Set<string>();
|
||||
for (const targets of adjacency.values()) {
|
||||
for (const target of targets) {
|
||||
hasIncoming.add(target);
|
||||
}
|
||||
}
|
||||
|
||||
const triggerNodes = nodes.filter(n => {
|
||||
if (EXCLUDED_TYPES.has(n.type)) return false;
|
||||
return !hasIncoming.has(n.name) || isTriggerType(n.type);
|
||||
});
|
||||
|
||||
// Pre-compute categories for this template (avoids re-parsing nodes_used per chain)
|
||||
let templateCategories: string[] = [];
|
||||
try {
|
||||
if (row.nodes_used) {
|
||||
const parsed = JSON.parse(row.nodes_used);
|
||||
if (Array.isArray(parsed)) {
|
||||
templateCategories = classifyTemplate(parsed.filter((t: string) => !EXCLUDED_TYPES.has(t)));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// skip
|
||||
}
|
||||
|
||||
// BFS from each trigger, extract chains of length 2-4
|
||||
for (const trigger of triggerNodes) {
|
||||
const queue: Array<{ nodeName: string; path: string[] }> = [
|
||||
{ nodeName: trigger.name, path: [nameToType.get(trigger.name)!] },
|
||||
];
|
||||
const visited = new Set<string>([trigger.name]);
|
||||
|
||||
while (queue.length > 0) {
|
||||
const { nodeName, path: currentPath } = queue.shift()!;
|
||||
|
||||
// Record chains of length 2, 3, 4
|
||||
if (currentPath.length >= 2 && currentPath.length <= 4) {
|
||||
const chainKey = currentPath.join('|||');
|
||||
globalChains.set(chainKey, (globalChains.get(chainKey) || 0) + 1);
|
||||
|
||||
for (const cat of templateCategories) {
|
||||
if (!categoryChains.has(cat)) {
|
||||
categoryChains.set(cat, new Map());
|
||||
}
|
||||
const catChainMap = categoryChains.get(cat)!;
|
||||
catChainMap.set(chainKey, (catChainMap.get(chainKey) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Stop extending at depth 4
|
||||
if (currentPath.length >= 4) continue;
|
||||
|
||||
const neighbors = adjacency.get(nodeName) || [];
|
||||
for (const neighbor of neighbors) {
|
||||
if (visited.has(neighbor)) continue;
|
||||
const neighborType = nameToType.get(neighbor);
|
||||
if (!neighborType) continue;
|
||||
visited.add(neighbor);
|
||||
queue.push({ nodeName: neighbor, path: [...currentPath, neighborType] });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pass2Time = ((Date.now() - pass2Start) / 1000).toFixed(1);
|
||||
console.log(`Pass 2 complete: ${pass2Time}s`);
|
||||
console.log(` Decompressed: ${decompressedCount}, Failed: ${decompressFailCount}`);
|
||||
console.log(` Unique edges: ${edgeFrequency.size}`);
|
||||
console.log(` Unique chains: ${globalChains.size}`);
|
||||
|
||||
// ---- Build output ----
|
||||
console.log('\n=== Building output ===');
|
||||
|
||||
// Global top nodes
|
||||
const topNodes: NodeFrequency[] = [...nodeFrequency.entries()]
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.slice(0, 50)
|
||||
.map(([type, count]) => ({
|
||||
type,
|
||||
count,
|
||||
frequency: Math.round((count / templateCount) * 100) / 100,
|
||||
displayName: getDisplayName(type),
|
||||
}));
|
||||
|
||||
// Global top edges
|
||||
const topEdges: EdgeFrequency[] = [...edgeFrequency.entries()]
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.slice(0, 50)
|
||||
.map(([key, count]) => {
|
||||
const [from, to] = key.split('|||');
|
||||
return { from, to, count };
|
||||
});
|
||||
|
||||
// Build category data
|
||||
const categories: Record<string, CategoryData> = {};
|
||||
|
||||
for (const [cat, templateIds] of categoryTemplates.entries()) {
|
||||
const catNodeMap = categoryNodes.get(cat)!;
|
||||
const catTemplateCount = templateIds.size;
|
||||
|
||||
// Top nodes for category, sorted by frequency
|
||||
const catTopNodes = [...catNodeMap.entries()]
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.slice(0, 20)
|
||||
.map(([type, count]) => ({
|
||||
type,
|
||||
frequency: Math.round((count / catTemplateCount) * 100) / 100,
|
||||
role: isTriggerType(type) ? 'trigger' : 'action',
|
||||
displayName: getDisplayName(type),
|
||||
}));
|
||||
|
||||
// Top chains for category
|
||||
const catChainMap = categoryChains.get(cat) || new Map<string, number>();
|
||||
const catTopChains: ChainFrequency[] = [...catChainMap.entries()]
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.slice(0, 10)
|
||||
.map(([chainKey, count]) => ({
|
||||
chain: chainKey.split('|||'),
|
||||
count,
|
||||
frequency: Math.round((count / catTemplateCount) * 100) / 100,
|
||||
}));
|
||||
|
||||
// Generate pattern string from top nodes
|
||||
// Order: triggers first, then transforms/logic, then actions
|
||||
const triggerNodes = catTopNodes.filter(n => n.role === 'trigger').slice(0, 1);
|
||||
const actionNodes = catTopNodes.filter(n => n.role !== 'trigger').slice(0, 3);
|
||||
const patternParts = [...triggerNodes, ...actionNodes].map(n => n.displayName);
|
||||
const pattern = patternParts.join(' \u2192 ') || 'Mixed workflow';
|
||||
|
||||
categories[cat] = {
|
||||
templateCount: catTemplateCount,
|
||||
pattern,
|
||||
nodes: catTopNodes,
|
||||
commonChains: catTopChains,
|
||||
};
|
||||
}
|
||||
|
||||
const output: PatternOutput = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
templateCount,
|
||||
categories,
|
||||
global: {
|
||||
topNodes,
|
||||
topEdges,
|
||||
},
|
||||
};
|
||||
|
||||
// Write output
|
||||
const outputPath = path.resolve(__dirname, '../../data/workflow-patterns.json');
|
||||
fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
|
||||
|
||||
console.log(`\nWritten ${Object.keys(categories).length} categories, ${templateCount} templates analyzed`);
|
||||
console.log(`Output: ${outputPath}`);
|
||||
console.log(`Pass 1: ${pass1Time}s, Pass 2: ${pass2Time}s`);
|
||||
console.log(`Total: ${((Date.now() - pass1Start) / 1000).toFixed(1)}s`);
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -151,7 +151,7 @@ describe('n8nDocumentationToolsFinal', () => {
|
||||
it('should have searchMode parameter with correct enum values', () => {
|
||||
const searchModeParam = tool?.inputSchema.properties?.searchMode;
|
||||
expect(searchModeParam).toBeDefined();
|
||||
expect(searchModeParam.enum).toEqual(['keyword', 'by_nodes', 'by_task', 'by_metadata']);
|
||||
expect(searchModeParam.enum).toEqual(['keyword', 'by_nodes', 'by_task', 'by_metadata', 'patterns']);
|
||||
expect(searchModeParam.default).toBe('keyword');
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user