ros/n8n-mcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Complete overhaul to enhanced documentation-only MCP server

Browse Source 
- Removed all workflow execution capabilities per user requirements
- Implemented enhanced documentation extraction with operations and API mappings
- Fixed credential code extraction for all nodes
- Fixed package info extraction (name and version)
- Enhanced operations parser to handle n8n markdown format
- Fixed documentation search to prioritize app nodes over trigger nodes
- Comprehensive test coverage for Slack node extraction
- All node information now includes:
  - Complete operations list (42 for Slack)
  - API method mappings with documentation URLs
  - Source code and credential definitions
  - Package metadata
  - Related resources and templates

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
2025-06-08 11:07:51 +00:00
parent 887e98ca0b
commit 3d7fdeba02
48 changed files with 9247 additions and 11057 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/db/schema-v2.sql
View File

@@ -1,99 +0,0 @@
-- Main nodes table with documentation and examples
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_type TEXT UNIQUE NOT NULL, -- e.g., "n8n-nodes-base.if"
name TEXT NOT NULL, -- e.g., "If"
display_name TEXT, -- e.g., "If"
description TEXT, -- Brief description from node definition
category TEXT, -- e.g., "Core Nodes", "Flow"
subcategory TEXT, -- More specific categorization
icon TEXT, -- Icon identifier/path
-- Source code
source_code TEXT NOT NULL, -- Full node source code
credential_code TEXT, -- Credential type definitions
code_hash TEXT NOT NULL, -- Hash for change detection
code_length INTEGER NOT NULL, -- Source code size
-- Documentation
documentation_markdown TEXT, -- Full markdown documentation from n8n-docs
documentation_url TEXT, -- URL to documentation page
-- Example usage
example_workflow TEXT, -- JSON example workflow using this node
example_parameters TEXT, -- JSON example of node parameters
properties_schema TEXT, -- JSON schema of node properties
-- Metadata
package_name TEXT NOT NULL, -- e.g., "n8n-nodes-base"
version TEXT, -- Node version
codex_data TEXT, -- Additional codex/metadata JSON
aliases TEXT, -- JSON array of alternative names
-- Flags
has_credentials INTEGER DEFAULT 0,
is_trigger INTEGER DEFAULT 0, -- Whether it's a trigger node
is_webhook INTEGER DEFAULT 0, -- Whether it's a webhook node
-- Timestamps
extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Indexes for performance
CREATE INDEX IF NOT EXISTS idx_nodes_package_name ON nodes(package_name);
CREATE INDEX IF NOT EXISTS idx_nodes_category ON nodes(category);
CREATE INDEX IF NOT EXISTS idx_nodes_code_hash ON nodes(code_hash);
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
CREATE INDEX IF NOT EXISTS idx_nodes_is_trigger ON nodes(is_trigger);
-- Full Text Search virtual table for comprehensive search
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
node_type,
name,
display_name,
description,
category,
documentation_markdown,
aliases,
content=nodes,
content_rowid=id
);
-- Triggers to keep FTS in sync
CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes
BEGIN
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases);
END;
CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes
BEGIN
DELETE FROM nodes_fts WHERE rowid = old.id;
END;
CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes
BEGIN
DELETE FROM nodes_fts WHERE rowid = old.id;
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases);
END;
-- Table for storing node documentation versions
CREATE TABLE IF NOT EXISTS documentation_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL, -- 'n8n-docs-repo', 'inline', 'generated'
commit_hash TEXT, -- Git commit hash if from repo
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Statistics table
CREATE TABLE IF NOT EXISTS extraction_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT,
total_nodes INTEGER NOT NULL,
nodes_with_docs INTEGER NOT NULL,
nodes_with_examples INTEGER NOT NULL,
total_code_size INTEGER NOT NULL,
total_docs_size INTEGER NOT NULL,
extraction_date DATETIME DEFAULT CURRENT_TIMESTAMP
);

112
src/db/schema.sql
View File

@@ -1,34 +1,75 @@
-- Main nodes table
-- Enhanced n8n Node Documentation Database Schema
-- This schema stores comprehensive node information including source code,
-- documentation, operations, API methods, examples, and metadata
-- Main nodes table with rich documentation
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_type TEXT UNIQUE NOT NULL,
name TEXT NOT NULL,
package_name TEXT NOT NULL,
display_name TEXT,
description TEXT,
code_hash TEXT NOT NULL,
code_length INTEGER NOT NULL,
source_location TEXT NOT NULL,
category TEXT,
subcategory TEXT,
icon TEXT,
-- Source code
source_code TEXT NOT NULL,
credential_code TEXT,
package_info TEXT, -- JSON
code_hash TEXT NOT NULL,
code_length INTEGER NOT NULL,
-- Documentation
documentation_markdown TEXT,
documentation_url TEXT,
documentation_title TEXT,
-- Enhanced documentation fields (stored as JSON)
operations TEXT,
api_methods TEXT,
documentation_examples TEXT,
templates TEXT,
related_resources TEXT,
required_scopes TEXT,
-- Example usage
example_workflow TEXT,
example_parameters TEXT,
properties_schema TEXT,
-- Metadata
package_name TEXT NOT NULL,
version TEXT,
codex_data TEXT,
aliases TEXT,
-- Flags
has_credentials INTEGER DEFAULT 0,
is_trigger INTEGER DEFAULT 0,
is_webhook INTEGER DEFAULT 0,
-- Timestamps
extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Indexes for performance
CREATE INDEX IF NOT EXISTS idx_nodes_package_name ON nodes(package_name);
CREATE INDEX IF NOT EXISTS idx_nodes_category ON nodes(category);
CREATE INDEX IF NOT EXISTS idx_nodes_code_hash ON nodes(code_hash);
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
CREATE INDEX IF NOT EXISTS idx_nodes_is_trigger ON nodes(is_trigger);
CREATE INDEX IF NOT EXISTS idx_nodes_has_credentials ON nodes(has_credentials);
-- Full Text Search virtual table for node search
-- Full Text Search table
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
node_type,
name,
display_name,
description,
package_name,
category,
documentation_markdown,
aliases,
content=nodes,
content_rowid=id
);
@@ -36,8 +77,8 @@ CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
-- Triggers to keep FTS in sync
CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes
BEGIN
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases);
END;
CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes
@@ -48,16 +89,55 @@ END;
CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes
BEGIN
DELETE FROM nodes_fts WHERE rowid = old.id;
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases);
END;
-- Statistics table for metadata
-- Documentation sources tracking
CREATE TABLE IF NOT EXISTS documentation_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
commit_hash TEXT,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Statistics tracking
CREATE TABLE IF NOT EXISTS extraction_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT,
total_nodes INTEGER NOT NULL,
total_packages INTEGER NOT NULL,
nodes_with_docs INTEGER NOT NULL,
nodes_with_examples INTEGER NOT NULL,
total_code_size INTEGER NOT NULL,
nodes_with_credentials INTEGER NOT NULL,
total_docs_size INTEGER NOT NULL,
extraction_date DATETIME DEFAULT CURRENT_TIMESTAMP
);
);
-- Views for common queries
CREATE VIEW IF NOT EXISTS nodes_summary AS
SELECT
node_type,
name,
display_name,
description,
category,
package_name,
CASE WHEN documentation_markdown IS NOT NULL THEN 1 ELSE 0 END as has_documentation,
CASE WHEN documentation_examples IS NOT NULL THEN 1 ELSE 0 END as has_examples,
CASE WHEN operations IS NOT NULL THEN 1 ELSE 0 END as has_operations,
has_credentials,
is_trigger,
is_webhook
FROM nodes;
CREATE VIEW IF NOT EXISTS package_summary AS
SELECT
package_name,
COUNT(*) as node_count,
SUM(CASE WHEN documentation_markdown IS NOT NULL THEN 1 ELSE 0 END) as nodes_with_docs,
SUM(CASE WHEN documentation_examples IS NOT NULL THEN 1 ELSE 0 END) as nodes_with_examples,
SUM(has_credentials) as nodes_with_credentials,
SUM(is_trigger) as trigger_nodes,
SUM(is_webhook) as webhook_nodes
FROM nodes
GROUP BY package_name
ORDER BY node_count DESC;

101
src/index-http.ts
View File

@@ -1,101 +0,0 @@
#!/usr/bin/env node
/**
* n8n Documentation MCP Server
* Copyright (c) 2025 n8n-mcp contributors
*
* This software is licensed under the Sustainable Use License.
* See the LICENSE file in the root directory of this source tree.
*/
import dotenv from 'dotenv';
import { N8NDocumentationRemoteServer } from './mcp/remote-server';
import { logger } from './utils/logger';
import * as path from 'path';
// Load environment variables
dotenv.config();
async function main() {
try {
// Get configuration from environment
const config = {
port: parseInt(process.env.MCP_PORT || '3000', 10),
host: process.env.MCP_HOST || '0.0.0.0',
domain: process.env.MCP_DOMAIN || 'localhost',
authToken: process.env.MCP_AUTH_TOKEN,
cors: process.env.MCP_CORS === 'true',
tlsCert: process.env.MCP_TLS_CERT,
tlsKey: process.env.MCP_TLS_KEY,
};
// Validate required configuration
if (!config.domain || config.domain === 'localhost') {
logger.warn('MCP_DOMAIN not set or set to localhost. Using default: localhost');
logger.warn('For production, set MCP_DOMAIN to your actual domain (e.g., n8ndocumentation.aiservices.pl)');
}
if (!config.authToken) {
logger.warn('MCP_AUTH_TOKEN not set. Server will run without authentication.');
logger.warn('For production, set MCP_AUTH_TOKEN to a secure value.');
}
// Set database path if not already set
if (!process.env.NODE_DB_PATH) {
process.env.NODE_DB_PATH = path.join(__dirname, '../data/nodes-v2.db');
}
logger.info('Starting n8n Documentation MCP Remote Server');
logger.info('Configuration:', {
port: config.port,
host: config.host,
domain: config.domain,
cors: config.cors,
authEnabled: !!config.authToken,
tlsEnabled: !!(config.tlsCert && config.tlsKey),
databasePath: process.env.NODE_DB_PATH,
});
const server = new N8NDocumentationRemoteServer(config);
// Start the server
await server.start();
// Handle graceful shutdown
const shutdown = async () => {
logger.info('Received shutdown signal');
await server.stop();
process.exit(0);
};
process.on('SIGINT', shutdown);
process.on('SIGTERM', shutdown);
logger.info('Server is ready to accept connections');
logger.info(`Claude Desktop configuration:`);
logger.info(JSON.stringify({
"mcpServers": {
"n8n-nodes-remote": {
"command": "curl",
"args": [
"-X", "POST",
"-H", "Content-Type: application/json",
"-H", `Authorization: Bearer ${config.authToken || 'YOUR_AUTH_TOKEN'}`,
"-d", "@-",
`https://${config.domain}/mcp`
],
"env": {}
}
}
}, null, 2));
} catch (error) {
logger.error('Failed to start server:', error);
process.exit(1);
}
}
// Run the server
main().catch((error) => {
logger.error('Unhandled error:', error);
process.exit(1);
});

67
src/index-v2.ts
View File

@@ -1,67 +0,0 @@
#!/usr/bin/env node
/**
* n8n Documentation MCP Server
* Copyright (c) 2025 n8n-mcp contributors
*
* This software is licensed under the Sustainable Use License.
* See the LICENSE file in the root directory of this source tree.
*/
import dotenv from 'dotenv';
import { N8NDocumentationMCPServer } from './mcp/server-v2';
import { MCPServerConfig } from './types';
import { logger } from './utils/logger';
import { NodeDocumentationService } from './services/node-documentation-service';
// Load environment variables
dotenv.config();
async function main() {
const config: MCPServerConfig = {
port: parseInt(process.env.MCP_SERVER_PORT || '3000', 10),
host: process.env.MCP_SERVER_HOST || 'localhost',
authToken: process.env.MCP_AUTH_TOKEN,
};
// Check if we should rebuild the database on startup
const rebuildOnStart = process.env.REBUILD_ON_START === 'true';
if (rebuildOnStart) {
logger.info('Rebuilding database on startup...');
const service = new NodeDocumentationService();
try {
const stats = await service.rebuildDatabase();
logger.info('Database rebuild complete:', stats);
} catch (error) {
logger.error('Failed to rebuild database:', error);
process.exit(1);
} finally {
service.close();
}
}
const server = new N8NDocumentationMCPServer(config);
try {
await server.start();
} catch (error) {
logger.error('Failed to start MCP server:', error);
process.exit(1);
}
}
// Handle graceful shutdown
process.on('SIGINT', () => {
logger.info('Received SIGINT, shutting down MCP server...');
process.exit(0);
});
process.on('SIGTERM', () => {
logger.info('Received SIGTERM, shutting down MCP server...');
process.exit(0);
});
main().catch((error) => {
logger.error('Unhandled error:', error);
process.exit(1);
});

592
src/mcp/http-server.ts
View File

@@ -1,592 +0,0 @@
import express from 'express';
import { createServer } from 'http';
import { WebSocketServer, WebSocket } from 'ws';
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
// WebSocketServerTransport is not available in the SDK, we'll implement a custom solution
import {
CallToolRequestSchema,
ErrorCode,
ListResourcesRequestSchema,
ListToolsRequestSchema,
McpError,
ReadResourceRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { nodeDocumentationTools } from './tools-v2';
import { logger } from '../utils/logger';
import { authenticateRequest } from '../utils/auth-middleware';
import * as crypto from 'crypto';
interface HttpServerConfig {
port: number;
host: string;
domain: string;
authToken?: string;
cors?: boolean;
tlsCert?: string;
tlsKey?: string;
}
/**
* HTTP/WebSocket MCP Server for remote access
*/
export class N8NDocumentationHttpServer {
private app: express.Application;
private server: any;
private wss!: WebSocketServer;
private nodeService: NodeDocumentationService;
private config: HttpServerConfig;
private activeSessions: Map<string, any> = new Map();
constructor(config: HttpServerConfig) {
this.config = config;
this.app = express();
this.nodeService = new NodeDocumentationService();
this.setupMiddleware();
this.setupRoutes();
this.setupWebSocket();
}
private setupMiddleware(): void {
// JSON parsing
this.app.use(express.json());
// CORS if enabled
if (this.config.cors) {
this.app.use((req, res, next): void => {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
if (req.method === 'OPTIONS') {
res.sendStatus(200);
return;
}
next();
});
}
// Request logging
this.app.use((req, res, next): void => {
logger.info(`${req.method} ${req.path}`, {
ip: req.ip,
userAgent: req.get('user-agent')
});
next();
});
}
private setupRoutes(): void {
// Health check endpoint
this.app.get('/health', (req, res) => {
res.json({
status: 'ok',
service: 'n8n-documentation-mcp',
version: '2.0.0',
uptime: process.uptime()
});
});
// MCP info endpoint
this.app.get('/mcp', (req, res) => {
res.json({
name: 'n8n-node-documentation',
version: '2.0.0',
description: 'MCP server providing n8n node documentation and source code',
transport: 'websocket',
endpoint: `wss://${this.config.domain}/mcp/websocket`,
authentication: 'bearer-token',
tools: nodeDocumentationTools.map(t => ({
name: t.name,
description: t.description
}))
});
});
// Database stats endpoint (public)
this.app.get('/stats', async (req, res) => {
try {
const stats = this.nodeService.getStatistics();
res.json(stats);
} catch (error) {
logger.error('Failed to get statistics:', error);
res.status(500).json({ error: 'Failed to retrieve statistics' });
}
});
// Rebuild endpoint (requires auth)
this.app.post('/rebuild', authenticateRequest(this.config.authToken), async (req, res) => {
try {
logger.info('Database rebuild requested');
const stats = await this.nodeService.rebuildDatabase();
res.json({
message: 'Database rebuild complete',
stats
});
} catch (error) {
logger.error('Rebuild failed:', error);
res.status(500).json({ error: 'Rebuild failed' });
}
});
}
private setupWebSocket(): void {
// Create HTTP server
this.server = createServer(this.app);
// Create WebSocket server
this.wss = new WebSocketServer({
server: this.server,
path: '/mcp/websocket'
});
this.wss.on('connection', async (ws: WebSocket, req: any) => {
const sessionId = crypto.randomUUID();
logger.info(`WebSocket connection established: ${sessionId}`);
// Authenticate WebSocket connection
const authHeader = req.headers.authorization;
if (this.config.authToken && authHeader !== `Bearer ${this.config.authToken}`) {
logger.warn(`Unauthorized WebSocket connection attempt: ${sessionId}`);
ws.close(1008, 'Unauthorized');
return;
}
try {
// Create MCP server instance for this connection
const mcpServer = new Server(
{
name: 'n8n-node-documentation',
version: '2.0.0',
},
{
capabilities: {
tools: {},
resources: {},
},
}
);
// Setup MCP handlers
this.setupMcpHandlers(mcpServer);
// WebSocket transport not available in SDK - implement JSON-RPC over WebSocket
// For now, we'll handle messages directly
ws.on('message', async (data: Buffer) => {
try {
const request = JSON.parse(data.toString());
// Process request through MCP server handlers
// This would need custom implementation
logger.warn('WebSocket MCP not fully implemented yet');
ws.send(JSON.stringify({
jsonrpc: '2.0',
id: request.id,
error: {
code: -32601,
message: 'WebSocket transport not implemented'
}
}));
} catch (error) {
logger.error('WebSocket message error:', error);
}
});
this.activeSessions.set(sessionId, { mcpServer, ws });
logger.info(`MCP session established: ${sessionId}`);
// Handle disconnect
ws.on('close', () => {
logger.info(`WebSocket connection closed: ${sessionId}`);
this.activeSessions.delete(sessionId);
});
} catch (error) {
logger.error(`Failed to establish MCP session: ${sessionId}`, error);
ws.close(1011, 'Server error');
}
});
}
private setupMcpHandlers(server: Server): void {
// List available tools
server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: nodeDocumentationTools,
}));
// List available resources
server.setRequestHandler(ListResourcesRequestSchema, async () => ({
resources: [
{
uri: 'nodes://list',
name: 'Available n8n Nodes',
description: 'List of all available n8n nodes',
mimeType: 'application/json',
},
{
uri: 'nodes://statistics',
name: 'Database Statistics',
description: 'Statistics about the node documentation database',
mimeType: 'application/json',
},
],
}));
// Read resources
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
const { uri } = request.params;
try {
if (uri === 'nodes://list') {
const nodes = await this.nodeService.listNodes();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(nodes.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
if (uri === 'nodes://statistics') {
const stats = this.nodeService.getStatistics();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(stats, null, 2),
},
],
};
}
throw new McpError(ErrorCode.InvalidRequest, `Unknown resource: ${uri}`);
} catch (error) {
logger.error('Resource read error:', error);
throw error instanceof McpError ? error : new McpError(
ErrorCode.InternalError,
`Failed to read resource: ${error instanceof Error ? error.message : String(error)}`
);
}
});
// Handle tool calls
server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
try {
switch (name) {
case 'list_nodes':
return await this.handleListNodes(args);
case 'get_node_info':
return await this.handleGetNodeInfo(args);
case 'search_nodes':
return await this.handleSearchNodes(args);
case 'get_node_example':
return await this.handleGetNodeExample(args);
case 'get_node_source_code':
return await this.handleGetNodeSourceCode(args);
case 'get_node_documentation':
return await this.handleGetNodeDocumentation(args);
case 'rebuild_database':
return await this.handleRebuildDatabase(args);
case 'get_database_statistics':
return await this.handleGetStatistics();
default:
throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${name}`);
}
} catch (error) {
logger.error(`Tool execution error (${name}):`, error);
throw error instanceof McpError ? error : new McpError(
ErrorCode.InternalError,
`Tool execution failed: ${error instanceof Error ? error.message : String(error)}`
);
}
});
}
// Tool handlers (copied from server-v2.ts)
private async handleListNodes(args: any): Promise<any> {
const nodes = await this.nodeService.listNodes();
let filtered = nodes;
if (args.category) {
filtered = filtered.filter(n => n.category === args.category);
}
if (args.packageName) {
filtered = filtered.filter(n => n.packageName === args.packageName);
}
if (args.isTrigger !== undefined) {
filtered = filtered.filter(n => n.isTrigger === args.isTrigger);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
packageName: n.packageName,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
isTrigger: n.isTrigger,
isWebhook: n.isWebhook,
})), null, 2),
},
],
};
}
private async handleGetNodeInfo(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
nodeType: nodeInfo.nodeType,
name: nodeInfo.name,
displayName: nodeInfo.displayName,
description: nodeInfo.description,
category: nodeInfo.category,
packageName: nodeInfo.packageName,
sourceCode: nodeInfo.sourceCode,
credentialCode: nodeInfo.credentialCode,
documentation: nodeInfo.documentation,
documentationUrl: nodeInfo.documentationUrl,
exampleWorkflow: nodeInfo.exampleWorkflow,
exampleParameters: nodeInfo.exampleParameters,
propertiesSchema: nodeInfo.propertiesSchema,
isTrigger: nodeInfo.isTrigger,
isWebhook: nodeInfo.isWebhook,
}, null, 2),
},
],
};
}
private async handleSearchNodes(args: any): Promise<any> {
if (!args.query) {
throw new McpError(ErrorCode.InvalidParams, 'query is required');
}
const results = await this.nodeService.searchNodes({
query: args.query,
category: args.category,
limit: args.limit || 20,
});
let filtered = results;
if (args.hasDocumentation) {
filtered = filtered.filter(n => !!n.documentation);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
private async handleGetNodeExample(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.exampleWorkflow) {
return {
content: [
{
type: 'text',
text: `No example available for node: ${args.nodeType}`,
},
],
};
}
return {
content: [
{
type: 'text',
text: JSON.stringify(nodeInfo.exampleWorkflow, null, 2),
},
],
};
}
private async handleGetNodeSourceCode(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
const response: any = {
nodeType: nodeInfo.nodeType,
sourceCode: nodeInfo.sourceCode,
};
if (args.includeCredentials && nodeInfo.credentialCode) {
response.credentialCode = nodeInfo.credentialCode;
}
return {
content: [
{
type: 'text',
text: JSON.stringify(response, null, 2),
},
],
};
}
private async handleGetNodeDocumentation(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.documentation) {
return {
content: [
{
type: 'text',
text: `No documentation available for node: ${args.nodeType}`,
},
],
};
}
const content = args.format === 'plain'
? nodeInfo.documentation.replace(/[#*`]/g, '')
: nodeInfo.documentation;
return {
content: [
{
type: 'text',
text: content,
},
],
};
}
private async handleRebuildDatabase(args: any): Promise<any> {
logger.info('Database rebuild requested via MCP');
const stats = await this.nodeService.rebuildDatabase();
return {
content: [
{
type: 'text',
text: JSON.stringify({
message: 'Database rebuild complete',
stats,
}, null, 2),
},
],
};
}
private async handleGetStatistics(): Promise<any> {
const stats = this.nodeService.getStatistics();
return {
content: [
{
type: 'text',
text: JSON.stringify(stats, null, 2),
},
],
};
}
async start(): Promise<void> {
return new Promise((resolve) => {
this.server.listen(this.config.port, this.config.host, () => {
logger.info(`n8n Documentation MCP HTTP server started`);
logger.info(`HTTP endpoint: http://${this.config.host}:${this.config.port}`);
logger.info(`WebSocket endpoint: ws://${this.config.host}:${this.config.port}/mcp/websocket`);
logger.info(`Domain: ${this.config.domain}`);
resolve();
});
});
}
async stop(): Promise<void> {
logger.info('Stopping n8n Documentation MCP HTTP server...');
// Close all WebSocket connections
this.wss.clients.forEach((ws: WebSocket) => ws.close());
// Close HTTP server
return new Promise((resolve) => {
this.server.close(() => {
this.nodeService.close();
logger.info('Server stopped');
resolve();
});
});
}
}

608
src/mcp/remote-server.ts
View File

@@ -1,608 +0,0 @@
import express from 'express';
import { createServer as createHttpServer } from 'http';
import { createServer as createHttpsServer } from 'https';
import {
ErrorCode,
McpError,
} from '@modelcontextprotocol/sdk/types.js';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { nodeDocumentationTools } from './tools-v2';
import { logger } from '../utils/logger';
import { authenticateRequest } from '../utils/auth-middleware';
import * as fs from 'fs';
interface RemoteServerConfig {
port: number;
host: string;
domain: string;
authToken?: string;
cors?: boolean;
tlsCert?: string;
tlsKey?: string;
}
/**
* Remote MCP Server using Streamable HTTP transport
* Based on MCP's modern approach for remote servers
*/
export class N8NDocumentationRemoteServer {
private app: express.Application;
private server: any;
private nodeService: NodeDocumentationService;
private config: RemoteServerConfig;
constructor(config: RemoteServerConfig) {
this.config = config;
this.app = express();
this.nodeService = new NodeDocumentationService();
this.setupMiddleware();
this.setupRoutes();
}
private setupMiddleware(): void {
// Parse JSON bodies with larger limit for MCP messages
this.app.use(express.json({ limit: '10mb' }));
// CORS if enabled
if (this.config.cors) {
this.app.use((req, res, next): void => {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization, X-Request-ID');
if (req.method === 'OPTIONS') {
res.sendStatus(200);
return;
}
next();
});
}
// Request logging
this.app.use((req, res, next): void => {
logger.info(`${req.method} ${req.path}`, {
ip: req.ip,
userAgent: req.get('user-agent'),
requestId: req.get('X-Request-ID')
});
next();
});
}
private setupRoutes(): void {
// Health check endpoint
this.app.get('/health', (req, res) => {
res.json({
status: 'ok',
service: 'n8n-documentation-mcp',
version: '2.0.0',
uptime: process.uptime(),
domain: this.config.domain
});
});
// MCP info endpoint - provides server capabilities
this.app.get('/', (req, res) => {
res.json({
name: 'n8n-node-documentation',
version: '2.0.0',
description: 'MCP server providing n8n node documentation and source code',
transport: 'http',
endpoint: `https://${this.config.domain}/mcp`,
authentication: this.config.authToken ? 'bearer-token' : 'none',
capabilities: {
tools: nodeDocumentationTools.map(t => ({
name: t.name,
description: t.description
})),
resources: [
{
uri: 'nodes://list',
name: 'Available n8n Nodes',
description: 'List of all available n8n nodes',
},
{
uri: 'nodes://statistics',
name: 'Database Statistics',
description: 'Statistics about the node documentation database',
},
]
}
});
});
// Database stats endpoint (public)
this.app.get('/stats', async (req, res) => {
try {
const stats = this.nodeService.getStatistics();
res.json(stats);
} catch (error) {
logger.error('Failed to get statistics:', error);
res.status(500).json({ error: 'Failed to retrieve statistics' });
}
});
// Rebuild endpoint (requires auth)
this.app.post('/rebuild', authenticateRequest(this.config.authToken), async (req, res) => {
try {
logger.info('Database rebuild requested');
const stats = await this.nodeService.rebuildDatabase();
res.json({
message: 'Database rebuild complete',
stats
});
} catch (error) {
logger.error('Rebuild failed:', error);
res.status(500).json({ error: 'Rebuild failed' });
}
});
// Main MCP endpoint - handles all MCP protocol messages
this.app.post('/mcp', authenticateRequest(this.config.authToken), async (req, res) => {
const requestId = req.get('X-Request-ID') || 'unknown';
try {
// Process the JSON-RPC request directly
const response = await this.handleJsonRpcRequest(req.body);
res.json(response);
} catch (error) {
logger.error(`MCP request failed (${requestId}):`, error);
if (!res.headersSent) {
res.status(500).json({
jsonrpc: '2.0',
id: req.body?.id || null,
error: {
code: -32603,
message: 'Internal error',
data: error instanceof Error ? error.message : String(error)
}
});
}
}
});
}
private async handleJsonRpcRequest(request: any): Promise<any> {
const { jsonrpc, method, params, id } = request;
if (jsonrpc !== '2.0') {
return {
jsonrpc: '2.0',
id: id || null,
error: {
code: -32600,
message: 'Invalid Request',
data: 'JSON-RPC version must be "2.0"'
}
};
}
try {
let result;
switch (method) {
case 'tools/list':
result = await this.handleListTools();
break;
case 'resources/list':
result = await this.handleListResources();
break;
case 'resources/read':
result = await this.handleReadResource(params);
break;
case 'tools/call':
result = await this.handleToolCall(params);
break;
default:
return {
jsonrpc: '2.0',
id: id || null,
error: {
code: -32601,
message: 'Method not found',
data: `Unknown method: ${method}`
}
};
}
return {
jsonrpc: '2.0',
id: id || null,
result
};
} catch (error) {
logger.error(`Error handling method ${method}:`, error);
const errorCode = error instanceof McpError ? error.code : -32603;
const errorMessage = error instanceof Error ? error.message : 'Internal error';
return {
jsonrpc: '2.0',
id: id || null,
error: {
code: errorCode,
message: errorMessage,
data: error instanceof McpError ? error.data : undefined
}
};
}
}
private async handleListTools(): Promise<any> {
return {
tools: nodeDocumentationTools,
};
}
private async handleListResources(): Promise<any> {
return {
resources: [
{
uri: 'nodes://list',
name: 'Available n8n Nodes',
description: 'List of all available n8n nodes',
mimeType: 'application/json',
},
{
uri: 'nodes://statistics',
name: 'Database Statistics',
description: 'Statistics about the node documentation database',
mimeType: 'application/json',
},
],
};
}
private async handleReadResource(params: any): Promise<any> {
const { uri } = params;
if (uri === 'nodes://list') {
const nodes = await this.nodeService.listNodes();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(nodes.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
if (uri === 'nodes://statistics') {
const stats = this.nodeService.getStatistics();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(stats, null, 2),
},
],
};
}
throw new McpError(ErrorCode.InvalidRequest, `Unknown resource: ${uri}`);
}
private async handleToolCall(params: any): Promise<any> {
const { name, arguments: args } = params;
switch (name) {
case 'list_nodes':
return await this.handleListNodes(args);
case 'get_node_info':
return await this.handleGetNodeInfo(args);
case 'search_nodes':
return await this.handleSearchNodes(args);
case 'get_node_example':
return await this.handleGetNodeExample(args);
case 'get_node_source_code':
return await this.handleGetNodeSourceCode(args);
case 'get_node_documentation':
return await this.handleGetNodeDocumentation(args);
case 'rebuild_database':
return await this.handleRebuildDatabase(args);
case 'get_database_statistics':
return await this.handleGetStatistics();
default:
throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${name}`);
}
}
// Tool handlers
private async handleListNodes(args: any): Promise<any> {
const nodes = await this.nodeService.listNodes();
let filtered = nodes;
if (args.category) {
filtered = filtered.filter(n => n.category === args.category);
}
if (args.packageName) {
filtered = filtered.filter(n => n.packageName === args.packageName);
}
if (args.isTrigger !== undefined) {
filtered = filtered.filter(n => n.isTrigger === args.isTrigger);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
packageName: n.packageName,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
isTrigger: n.isTrigger,
isWebhook: n.isWebhook,
})), null, 2),
},
],
};
}
private async handleGetNodeInfo(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
nodeType: nodeInfo.nodeType,
name: nodeInfo.name,
displayName: nodeInfo.displayName,
description: nodeInfo.description,
category: nodeInfo.category,
packageName: nodeInfo.packageName,
sourceCode: nodeInfo.sourceCode,
credentialCode: nodeInfo.credentialCode,
documentation: nodeInfo.documentation,
documentationUrl: nodeInfo.documentationUrl,
exampleWorkflow: nodeInfo.exampleWorkflow,
exampleParameters: nodeInfo.exampleParameters,
propertiesSchema: nodeInfo.propertiesSchema,
isTrigger: nodeInfo.isTrigger,
isWebhook: nodeInfo.isWebhook,
}, null, 2),
},
],
};
}
private async handleSearchNodes(args: any): Promise<any> {
if (!args.query) {
throw new McpError(ErrorCode.InvalidParams, 'query is required');
}
const results = await this.nodeService.searchNodes({
query: args.query,
category: args.category,
limit: args.limit || 20,
});
let filtered = results;
if (args.hasDocumentation) {
filtered = filtered.filter(n => !!n.documentation);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
private async handleGetNodeExample(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.exampleWorkflow) {
return {
content: [
{
type: 'text',
text: `No example available for node: ${args.nodeType}`,
},
],
};
}
return {
content: [
{
type: 'text',
text: JSON.stringify(nodeInfo.exampleWorkflow, null, 2),
},
],
};
}
private async handleGetNodeSourceCode(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
const response: any = {
nodeType: nodeInfo.nodeType,
sourceCode: nodeInfo.sourceCode,
};
if (args.includeCredentials && nodeInfo.credentialCode) {
response.credentialCode = nodeInfo.credentialCode;
}
return {
content: [
{
type: 'text',
text: JSON.stringify(response, null, 2),
},
],
};
}
private async handleGetNodeDocumentation(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.documentation) {
return {
content: [
{
type: 'text',
text: `No documentation available for node: ${args.nodeType}`,
},
],
};
}
const content = args.format === 'plain'
? nodeInfo.documentation.replace(/[#*`]/g, '')
: nodeInfo.documentation;
return {
content: [
{
type: 'text',
text: content,
},
],
};
}
private async handleRebuildDatabase(args: any): Promise<any> {
logger.info('Database rebuild requested via MCP');
const stats = await this.nodeService.rebuildDatabase();
return {
content: [
{
type: 'text',
text: JSON.stringify({
message: 'Database rebuild complete',
stats,
}, null, 2),
},
],
};
}
private async handleGetStatistics(): Promise<any> {
const stats = this.nodeService.getStatistics();
return {
content: [
{
type: 'text',
text: JSON.stringify(stats, null, 2),
},
],
};
}
async start(): Promise<void> {
// Create server (HTTP or HTTPS)
if (this.config.tlsCert && this.config.tlsKey) {
const tlsOptions = {
cert: fs.readFileSync(this.config.tlsCert),
key: fs.readFileSync(this.config.tlsKey),
};
this.server = createHttpsServer(tlsOptions, this.app);
} else {
this.server = createHttpServer(this.app);
}
return new Promise((resolve) => {
this.server.listen(this.config.port, this.config.host, () => {
const protocol = this.config.tlsCert ? 'https' : 'http';
logger.info(`n8n Documentation MCP Remote server started`);
logger.info(`Endpoint: ${protocol}://${this.config.host}:${this.config.port}`);
logger.info(`Domain: ${this.config.domain}`);
logger.info(`MCP endpoint: ${protocol}://${this.config.domain}/mcp`);
resolve();
});
});
}
async stop(): Promise<void> {
logger.info('Stopping n8n Documentation MCP Remote server...');
return new Promise((resolve) => {
this.server.close(() => {
this.nodeService.close();
logger.info('Server stopped');
resolve();
});
});
}
}

435
src/mcp/server-v2.ts
View File

@@ -1,435 +0,0 @@
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
CallToolRequestSchema,
ErrorCode,
ListResourcesRequestSchema,
ListToolsRequestSchema,
McpError,
ReadResourceRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { nodeDocumentationTools } from './tools-v2';
import { logger } from '../utils/logger';
import { MCPServerConfig } from '../types';
/**
* MCP Server focused on serving n8n node documentation and code
*/
export class N8NDocumentationMCPServer {
private server: Server;
private nodeService: NodeDocumentationService;
constructor(config: MCPServerConfig) {
logger.info('Initializing n8n Documentation MCP server', { config });
this.server = new Server(
{
name: 'n8n-node-documentation',
version: '2.0.0',
},
{
capabilities: {
tools: {},
resources: {},
},
}
);
this.nodeService = new NodeDocumentationService();
this.setupHandlers();
}
private setupHandlers(): void {
// List available tools
this.server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: nodeDocumentationTools,
}));
// List available resources
this.server.setRequestHandler(ListResourcesRequestSchema, async () => ({
resources: [
{
uri: 'nodes://list',
name: 'Available n8n Nodes',
description: 'List of all available n8n nodes',
mimeType: 'application/json',
},
{
uri: 'nodes://statistics',
name: 'Database Statistics',
description: 'Statistics about the node documentation database',
mimeType: 'application/json',
},
],
}));
// Read resources
this.server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
const { uri } = request.params;
try {
if (uri === 'nodes://list') {
const nodes = await this.nodeService.listNodes();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(nodes.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
if (uri === 'nodes://statistics') {
const stats = this.nodeService.getStatistics();
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(stats, null, 2),
},
],
};
}
// Handle specific node URIs like nodes://info/n8n-nodes-base.if
const nodeMatch = uri.match(/^nodes:\/\/info\/(.+)$/);
if (nodeMatch) {
const nodeType = nodeMatch[1];
const nodeInfo = await this.nodeService.getNodeInfo(nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${nodeType}`);
}
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify(nodeInfo, null, 2),
},
],
};
}
throw new McpError(ErrorCode.InvalidRequest, `Unknown resource: ${uri}`);
} catch (error) {
logger.error('Resource read error:', error);
throw error instanceof McpError ? error : new McpError(
ErrorCode.InternalError,
`Failed to read resource: ${error instanceof Error ? error.message : String(error)}`
);
}
});
// Handle tool calls
this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
try {
switch (name) {
case 'list_nodes':
return await this.handleListNodes(args);
case 'get_node_info':
return await this.handleGetNodeInfo(args);
case 'search_nodes':
return await this.handleSearchNodes(args);
case 'get_node_example':
return await this.handleGetNodeExample(args);
case 'get_node_source_code':
return await this.handleGetNodeSourceCode(args);
case 'get_node_documentation':
return await this.handleGetNodeDocumentation(args);
case 'rebuild_database':
return await this.handleRebuildDatabase(args);
case 'get_database_statistics':
return await this.handleGetStatistics();
default:
throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${name}`);
}
} catch (error) {
logger.error(`Tool execution error (${name}):`, error);
throw error instanceof McpError ? error : new McpError(
ErrorCode.InternalError,
`Tool execution failed: ${error instanceof Error ? error.message : String(error)}`
);
}
});
}
private async handleListNodes(args: any): Promise<any> {
const nodes = await this.nodeService.listNodes();
// Apply filters
let filtered = nodes;
if (args.category) {
filtered = filtered.filter(n => n.category === args.category);
}
if (args.packageName) {
filtered = filtered.filter(n => n.packageName === args.packageName);
}
if (args.isTrigger !== undefined) {
filtered = filtered.filter(n => n.isTrigger === args.isTrigger);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
packageName: n.packageName,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
isTrigger: n.isTrigger,
isWebhook: n.isWebhook,
})), null, 2),
},
],
};
}
private async handleGetNodeInfo(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
nodeType: nodeInfo.nodeType,
name: nodeInfo.name,
displayName: nodeInfo.displayName,
description: nodeInfo.description,
category: nodeInfo.category,
packageName: nodeInfo.packageName,
sourceCode: nodeInfo.sourceCode,
credentialCode: nodeInfo.credentialCode,
documentation: nodeInfo.documentation,
documentationUrl: nodeInfo.documentationUrl,
exampleWorkflow: nodeInfo.exampleWorkflow,
exampleParameters: nodeInfo.exampleParameters,
propertiesSchema: nodeInfo.propertiesSchema,
isTrigger: nodeInfo.isTrigger,
isWebhook: nodeInfo.isWebhook,
}, null, 2),
},
],
};
}
private async handleSearchNodes(args: any): Promise<any> {
if (!args.query) {
throw new McpError(ErrorCode.InvalidParams, 'query is required');
}
const results = await this.nodeService.searchNodes({
query: args.query,
category: args.category,
limit: args.limit || 20,
});
// Filter by documentation if requested
let filtered = results;
if (args.hasDocumentation) {
filtered = filtered.filter(n => !!n.documentation);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(filtered.map(n => ({
nodeType: n.nodeType,
name: n.name,
displayName: n.displayName,
category: n.category,
description: n.description,
hasDocumentation: !!n.documentation,
hasExample: !!n.exampleWorkflow,
})), null, 2),
},
],
};
}
private async handleGetNodeExample(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.exampleWorkflow) {
return {
content: [
{
type: 'text',
text: `No example available for node: ${args.nodeType}`,
},
],
};
}
return {
content: [
{
type: 'text',
text: JSON.stringify(nodeInfo.exampleWorkflow, null, 2),
},
],
};
}
private async handleGetNodeSourceCode(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
const response: any = {
nodeType: nodeInfo.nodeType,
sourceCode: nodeInfo.sourceCode,
};
if (args.includeCredentials && nodeInfo.credentialCode) {
response.credentialCode = nodeInfo.credentialCode;
}
return {
content: [
{
type: 'text',
text: JSON.stringify(response, null, 2),
},
],
};
}
private async handleGetNodeDocumentation(args: any): Promise<any> {
if (!args.nodeType) {
throw new McpError(ErrorCode.InvalidParams, 'nodeType is required');
}
const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType);
if (!nodeInfo) {
throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`);
}
if (!nodeInfo.documentation) {
return {
content: [
{
type: 'text',
text: `No documentation available for node: ${args.nodeType}`,
},
],
};
}
const content = args.format === 'plain'
? nodeInfo.documentation.replace(/[#*`]/g, '')
: nodeInfo.documentation;
return {
content: [
{
type: 'text',
text: content,
},
],
};
}
private async handleRebuildDatabase(args: any): Promise<any> {
logger.info('Starting database rebuild...');
const stats = await this.nodeService.rebuildDatabase();
return {
content: [
{
type: 'text',
text: JSON.stringify({
message: 'Database rebuild complete',
stats,
}, null, 2),
},
],
};
}
private async handleGetStatistics(): Promise<any> {
const stats = this.nodeService.getStatistics();
return {
content: [
{
type: 'text',
text: JSON.stringify(stats, null, 2),
},
],
};
}
async start(): Promise<void> {
logger.info('Starting n8n Documentation MCP server...');
const transport = new StdioServerTransport();
await this.server.connect(transport);
logger.info('n8n Documentation MCP server started successfully');
}
async stop(): Promise<void> {
logger.info('Stopping n8n Documentation MCP server...');
await this.server.close();
this.nodeService.close();
logger.info('Server stopped');
}
}

149
src/mcp/server.ts
View File

@@ -16,18 +16,18 @@ import { N8NApiClient } from '../utils/n8n-client';
import { N8NMCPBridge } from '../utils/bridge';
import { logger } from '../utils/logger';
import { NodeSourceExtractor } from '../utils/node-source-extractor';
import { SQLiteStorageService } from '../services/sqlite-storage-service';
import { NodeDocumentationService } from '../services/node-documentation-service';
export class N8NMCPServer {
private server: Server;
private n8nClient: N8NApiClient;
private nodeExtractor: NodeSourceExtractor;
private nodeStorage: SQLiteStorageService;
private nodeDocService: NodeDocumentationService;
constructor(config: MCPServerConfig, n8nConfig: N8NConfig) {
this.n8nClient = new N8NApiClient(n8nConfig);
this.nodeExtractor = new NodeSourceExtractor();
this.nodeStorage = new SQLiteStorageService();
this.nodeDocService = new NodeDocumentationService();
logger.info('Initializing n8n MCP server', { config, n8nConfig });
this.server = new Server(
{
@@ -164,12 +164,14 @@ export class N8NMCPServer {
return this.getNodeSourceCode(args);
case 'list_available_nodes':
return this.listAvailableNodes(args);
case 'extract_all_nodes':
return this.extractAllNodes(args);
case 'get_node_info':
return this.getNodeInfo(args);
case 'search_nodes':
return this.searchNodes(args);
case 'get_node_statistics':
return this.getNodeStatistics(args);
case 'rebuild_documentation_database':
return this.rebuildDocumentationDatabase(args);
default:
throw new Error(`Unknown tool: ${name}`);
}
@@ -323,84 +325,87 @@ export class N8NMCPServer {
}
}
private async extractAllNodes(args: any): Promise<any> {
private async getNodeInfo(args: any): Promise<any> {
try {
logger.info(`Extracting all nodes`, args);
logger.info('Getting comprehensive node information', args);
const nodeInfo = await this.nodeDocService.getNodeInfo(args.nodeType);
// Get list of all nodes
const allNodes = await this.nodeExtractor.listAvailableNodes();
let nodesToExtract = allNodes;
// Apply filters
if (args.packageFilter) {
nodesToExtract = nodesToExtract.filter(node =>
node.packageName === args.packageFilter ||
node.location?.includes(args.packageFilter)
);
if (!nodeInfo) {
throw new Error(`Node ${args.nodeType} not found`);
}
if (args.limit) {
nodesToExtract = nodesToExtract.slice(0, args.limit);
}
logger.info(`Extracting ${nodesToExtract.length} nodes...`);
const extractedNodes = [];
const errors = [];
for (const node of nodesToExtract) {
try {
const nodeType = node.packageName ? `${node.packageName}.${node.name}` : node.name;
const nodeInfo = await this.nodeExtractor.extractNodeSource(nodeType);
await this.nodeStorage.storeNode(nodeInfo);
extractedNodes.push(nodeType);
} catch (error) {
errors.push({
node: node.name,
error: error instanceof Error ? error.message : 'Unknown error'
});
}
}
const stats = await this.nodeStorage.getStatistics();
return {
success: true,
extracted: extractedNodes.length,
failed: errors.length,
totalStored: stats.totalNodes,
errors: errors.slice(0, 10), // Limit error list
statistics: stats
nodeType: nodeInfo.nodeType,
name: nodeInfo.name,
displayName: nodeInfo.displayName,
description: nodeInfo.description,
category: nodeInfo.category,
subcategory: nodeInfo.subcategory,
icon: nodeInfo.icon,
documentation: {
markdown: nodeInfo.documentationMarkdown,
url: nodeInfo.documentationUrl,
title: nodeInfo.documentationTitle,
},
operations: nodeInfo.operations || [],
apiMethods: nodeInfo.apiMethods || [],
examples: nodeInfo.documentationExamples || [],
templates: nodeInfo.templates || [],
relatedResources: nodeInfo.relatedResources || [],
requiredScopes: nodeInfo.requiredScopes || [],
exampleWorkflow: nodeInfo.exampleWorkflow,
exampleParameters: nodeInfo.exampleParameters,
propertiesSchema: nodeInfo.propertiesSchema,
metadata: {
packageName: nodeInfo.packageName,
version: nodeInfo.version,
hasCredentials: nodeInfo.hasCredentials,
isTrigger: nodeInfo.isTrigger,
isWebhook: nodeInfo.isWebhook,
aliases: nodeInfo.aliases,
},
sourceCode: {
node: nodeInfo.sourceCode,
credential: nodeInfo.credentialCode,
},
};
} catch (error) {
logger.error(`Failed to extract all nodes`, error);
throw new Error(`Failed to extract all nodes: ${error instanceof Error ? error.message : 'Unknown error'}`);
logger.error(`Failed to get node info`, error);
throw new Error(`Failed to get node info: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
private async searchNodes(args: any): Promise<any> {
try {
logger.info(`Searching nodes`, args);
const results = await this.nodeStorage.searchNodes({
logger.info('Searching nodes with enhanced filtering', args);
const results = await this.nodeDocService.searchNodes({
query: args.query,
category: args.category,
packageName: args.packageName,
hasCredentials: args.hasCredentials,
limit: args.limit || 20
isTrigger: args.isTrigger,
limit: args.limit || 20,
});
return {
nodes: results.map(node => ({
nodeType: node.nodeType,
name: node.name,
packageName: node.packageName,
displayName: node.displayName,
description: node.description,
codeLength: node.codeLength,
hasCredentials: node.hasCredentials,
location: node.sourceLocation
category: node.category,
packageName: node.packageName,
hasDocumentation: !!node.documentationMarkdown,
hasExamples: !!(node.documentationExamples && node.documentationExamples.length > 0),
operationCount: node.operations?.length || 0,
metadata: {
hasCredentials: node.hasCredentials,
isTrigger: node.isTrigger,
isWebhook: node.isWebhook,
},
})),
total: results.length
total: results.length,
};
} catch (error) {
logger.error(`Failed to search nodes`, error);
@@ -411,12 +416,11 @@ export class N8NMCPServer {
private async getNodeStatistics(args: any): Promise<any> {
try {
logger.info(`Getting node statistics`);
const stats = await this.nodeStorage.getStatistics();
const stats = this.nodeDocService.getStatistics();
return {
...stats,
formattedTotalSize: `${(stats.totalCodeSize / 1024 / 1024).toFixed(2)} MB`,
formattedAverageSize: `${(stats.averageNodeSize / 1024).toFixed(2)} KB`
formattedTotalSize: stats.totalCodeSize ? `${(stats.totalCodeSize / 1024 / 1024).toFixed(2)} MB` : '0 MB',
};
} catch (error) {
logger.error(`Failed to get node statistics`, error);
@@ -424,6 +428,23 @@ export class N8NMCPServer {
}
}
private async rebuildDocumentationDatabase(args: any): Promise<any> {
try {
logger.info('Rebuilding documentation database', args);
const stats = await this.nodeDocService.rebuildDatabase();
return {
success: true,
message: 'Documentation database rebuilt successfully',
statistics: stats,
};
} catch (error) {
logger.error(`Failed to rebuild documentation database`, error);
throw new Error(`Failed to rebuild documentation database: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
async start(): Promise<void> {
try {
logger.info('Starting n8n MCP server...');

144
src/mcp/tools-v2.ts
View File

@@ -1,144 +0,0 @@
import { ToolDefinition } from '../types';
/**
* Simplified MCP tools focused on serving n8n node documentation and code
*/
export const nodeDocumentationTools: ToolDefinition[] = [
{
name: 'list_nodes',
description: 'List all available n8n nodes with basic information',
inputSchema: {
type: 'object',
properties: {
category: {
type: 'string',
description: 'Filter by category (e.g., "Core Nodes", "Flow", "Data Transformation")',
},
packageName: {
type: 'string',
description: 'Filter by package name (e.g., "n8n-nodes-base")',
},
isTrigger: {
type: 'boolean',
description: 'Filter to show only trigger nodes',
},
},
},
},
{
name: 'get_node_info',
description: 'Get complete information about a specific n8n node including source code, documentation, and examples',
inputSchema: {
type: 'object',
properties: {
nodeType: {
type: 'string',
description: 'The node type or name (e.g., "n8n-nodes-base.if", "If", "webhook")',
},
},
required: ['nodeType'],
},
},
{
name: 'search_nodes',
description: 'Search for n8n nodes by name, description, or documentation content',
inputSchema: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'Search query (searches in node names, descriptions, and documentation)',
},
category: {
type: 'string',
description: 'Filter by category',
},
hasDocumentation: {
type: 'boolean',
description: 'Filter to show only nodes with documentation',
},
limit: {
type: 'number',
description: 'Maximum number of results to return',
default: 20,
},
},
required: ['query'],
},
},
{
name: 'get_node_example',
description: 'Get example workflow/usage for a specific n8n node',
inputSchema: {
type: 'object',
properties: {
nodeType: {
type: 'string',
description: 'The node type or name',
},
},
required: ['nodeType'],
},
},
{
name: 'get_node_source_code',
description: 'Get only the source code of a specific n8n node',
inputSchema: {
type: 'object',
properties: {
nodeType: {
type: 'string',
description: 'The node type or name',
},
includeCredentials: {
type: 'boolean',
description: 'Include credential type definitions if available',
default: false,
},
},
required: ['nodeType'],
},
},
{
name: 'get_node_documentation',
description: 'Get only the documentation for a specific n8n node',
inputSchema: {
type: 'object',
properties: {
nodeType: {
type: 'string',
description: 'The node type or name',
},
format: {
type: 'string',
enum: ['markdown', 'plain'],
description: 'Documentation format',
default: 'markdown',
},
},
required: ['nodeType'],
},
},
{
name: 'rebuild_database',
description: 'Rebuild the entire node database with latest information from n8n and documentation',
inputSchema: {
type: 'object',
properties: {
includeDocumentation: {
type: 'boolean',
description: 'Include documentation from n8n-docs repository',
default: true,
},
},
},
},
{
name: 'get_database_statistics',
description: 'Get statistics about the node database',
inputSchema: {
type: 'object',
properties: {},
},
},
];

46
src/mcp/tools.ts
View File

@@ -182,31 +182,40 @@ export const n8nTools: ToolDefinition[] = [
},
},
{
name: 'extract_all_nodes',
description: 'Extract and store all available n8n nodes in the database',
name: 'get_node_statistics',
description: 'Get statistics about stored n8n nodes',
inputSchema: {
type: 'object',
properties: {},
},
},
{
name: 'get_node_info',
description: 'Get comprehensive information about a specific n8n node including documentation, operations, API methods, and examples',
inputSchema: {
type: 'object',
properties: {
packageFilter: {
nodeType: {
type: 'string',
description: 'Optional package name to filter extraction',
},
limit: {
type: 'number',
description: 'Maximum number of nodes to extract',
description: 'The node type identifier (e.g., n8n-nodes-base.slack)',
},
},
required: ['nodeType'],
},
},
{
name: 'search_nodes',
description: 'Search for n8n nodes by name, package, or functionality',
description: 'Search n8n nodes with full-text search and advanced filtering',
inputSchema: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'Search query',
description: 'Search query for full-text search',
},
category: {
type: 'string',
description: 'Filter by node category',
},
packageName: {
type: 'string',
@@ -214,7 +223,11 @@ export const n8nTools: ToolDefinition[] = [
},
hasCredentials: {
type: 'boolean',
description: 'Filter nodes that have credentials',
description: 'Filter nodes that require credentials',
},
isTrigger: {
type: 'boolean',
description: 'Filter trigger nodes only',
},
limit: {
type: 'number',
@@ -225,11 +238,16 @@ export const n8nTools: ToolDefinition[] = [
},
},
{
name: 'get_node_statistics',
description: 'Get statistics about stored n8n nodes',
name: 'rebuild_documentation_database',
description: 'Rebuild the node documentation database with the latest information',
inputSchema: {
type: 'object',
properties: {},
properties: {
packageFilter: {
type: 'string',
description: 'Optional: Only rebuild nodes from specific package',
},
},
},
},
];

67
src/scripts/rebuild-database-v2.ts
View File

@@ -1,67 +0,0 @@
#!/usr/bin/env node
import { NodeDocumentationService } from '../services/node-documentation-service';
import { logger } from '../utils/logger';
async function rebuildDatabase() {
console.log('🔄 Starting complete database rebuild...\n');
const service = new NodeDocumentationService();
try {
const startTime = Date.now();
console.log('1️⃣ Initializing services...');
console.log('2️⃣ Fetching n8n-docs repository...');
console.log('3️⃣ Discovering available nodes...');
console.log('4️⃣ Extracting node information...\n');
const stats = await service.rebuildDatabase();
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
console.log('\n📊 Rebuild Results:');
console.log(` Total nodes processed: ${stats.total}`);
console.log(` Successfully stored: ${stats.successful}`);
console.log(` Failed: ${stats.failed}`);
console.log(` Duration: ${duration}s`);
if (stats.errors.length > 0) {
console.log('\n⚠️ First 5 errors:');
stats.errors.slice(0, 5).forEach(error => {
console.log(` - ${error}`);
});
if (stats.errors.length > 5) {
console.log(` ... and ${stats.errors.length - 5} more errors`);
}
}
// Get final statistics
const dbStats = service.getStatistics();
console.log('\n📈 Database Statistics:');
console.log(` Total nodes: ${dbStats.totalNodes}`);
console.log(` Nodes with documentation: ${dbStats.nodesWithDocs}`);
console.log(` Nodes with examples: ${dbStats.nodesWithExamples}`);
console.log(` Trigger nodes: ${dbStats.triggerNodes}`);
console.log(` Webhook nodes: ${dbStats.webhookNodes}`);
console.log(` Total packages: ${dbStats.totalPackages}`);
console.log('\n✨ Database rebuild complete!');
} catch (error) {
console.error('\n❌ Database rebuild failed:', error);
process.exit(1);
} finally {
service.close();
}
}
// Run if called directly
if (require.main === module) {
rebuildDatabase().catch(error => {
console.error(error);
process.exit(1);
});
}
export { rebuildDatabase };

134
src/scripts/rebuild-database.ts
View File

@@ -1,129 +1,79 @@
#!/usr/bin/env node
import { NodeSourceExtractor } from '../utils/node-source-extractor';
import { SQLiteStorageService } from '../services/sqlite-storage-service';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { logger } from '../utils/logger';
import * as path from 'path';
/**
* Rebuild the entire nodes database by extracting all available nodes
* Rebuild the enhanced documentation database
*/
async function rebuildDatabase() {
console.log('🔄 Starting database rebuild...\n');
async function rebuildDocumentationDatabase() {
console.log('🔄 Starting enhanced documentation database rebuild...\n');
const startTime = Date.now();
const extractor = new NodeSourceExtractor();
const storage = new SQLiteStorageService();
const service = new NodeDocumentationService();
try {
// Step 1: Clear existing database
console.log('1️⃣ Clearing existing database...');
await storage.rebuildDatabase();
// Run the rebuild
const results = await service.rebuildDatabase();
// Step 2: Get all available nodes
console.log('2️⃣ Discovering available nodes...');
const allNodes = await extractor.listAvailableNodes();
console.log(` Found ${allNodes.length} nodes\n`);
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
// Step 3: Extract and store each node
console.log('3️⃣ Extracting and storing nodes...');
let processed = 0;
let stored = 0;
let failed = 0;
const errors: Array<{ node: string; error: string }> = [];
console.log('\n✅ Enhanced documentation database rebuild completed!\n');
console.log('📊 Results:');
console.log(` Total nodes found: ${results.total}`);
console.log(` Successfully processed: ${results.successful}`);
console.log(` Failed: ${results.failed}`);
console.log(` Duration: ${duration}s`);
// Process in batches for better performance
const batchSize = 50;
for (let i = 0; i < allNodes.length; i += batchSize) {
const batch = allNodes.slice(i, Math.min(i + batchSize, allNodes.length));
const nodeInfos = [];
if (results.errors.length > 0) {
console.log(`\n⚠️ First ${Math.min(5, results.errors.length)} errors:`);
results.errors.slice(0, 5).forEach(err => {
console.log(` - ${err}`);
});
for (const node of batch) {
processed++;
try {
const nodeType = node.packageName ? `${node.packageName}.${node.name}` : node.name;
// Show progress
if (processed % 100 === 0) {
const progress = ((processed / allNodes.length) * 100).toFixed(1);
console.log(` Progress: ${processed}/${allNodes.length} (${progress}%)`);
}
const nodeInfo = await extractor.extractNodeSource(nodeType);
nodeInfos.push(nodeInfo);
stored++;
} catch (error) {
failed++;
const errorMsg = error instanceof Error ? error.message : 'Unknown error';
errors.push({
node: node.name,
error: errorMsg
});
// Log first few errors
if (errors.length <= 5) {
logger.debug(`Failed to extract ${node.name}: ${errorMsg}`);
}
}
}
// Bulk store the batch
if (nodeInfos.length > 0) {
await storage.bulkStoreNodes(nodeInfos);
if (results.errors.length > 5) {
console.log(` ... and ${results.errors.length - 5} more errors`);
}
}
// Step 4: Save statistics
console.log('\n4️⃣ Saving statistics...');
const stats = await storage.getStatistics();
await storage.saveExtractionStats(stats);
// Step 5: Display results
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
console.log('\n✅ Database rebuild completed!\n');
console.log('📊 Results:');
console.log(` Total nodes found: ${allNodes.length}`);
console.log(` Successfully stored: ${stored}`);
console.log(` Failed: ${failed}`);
console.log(` Duration: ${duration}s`);
console.log(` Database size: ${(stats.totalCodeSize / 1024 / 1024).toFixed(2)} MB`);
// Get and display statistics
const stats = service.getStatistics();
console.log('\n📈 Database Statistics:');
console.log(` Total nodes: ${stats.totalNodes}`);
console.log(` Nodes with documentation: ${stats.nodesWithDocs}`);
console.log(` Nodes with examples: ${stats.nodesWithExamples}`);
console.log(` Nodes with credentials: ${stats.nodesWithCredentials}`);
console.log(` Trigger nodes: ${stats.triggerNodes}`);
console.log(` Webhook nodes: ${stats.webhookNodes}`);
console.log('\n📦 Package distribution:');
stats.packageDistribution.slice(0, 10).forEach(pkg => {
stats.packageDistribution.slice(0, 10).forEach((pkg: any) => {
console.log(` ${pkg.package}: ${pkg.count} nodes`);
});
if (errors.length > 0) {
console.log(`\n⚠️ First ${Math.min(5, errors.length)} errors:`);
errors.slice(0, 5).forEach(err => {
console.log(` - ${err.node}: ${err.error}`);
});
if (errors.length > 5) {
console.log(` ... and ${errors.length - 5} more errors`);
}
}
// Close database connection
storage.close();
service.close();
console.log('\n✨ Database is ready for use!');
console.log('\n✨ Enhanced documentation database is ready!');
console.log('💡 The database now includes:');
console.log(' - Complete node source code');
console.log(' - Enhanced documentation with operations and API methods');
console.log(' - Code examples and templates');
console.log(' - Related resources and required scopes');
} catch (error) {
console.error('\n❌ Database rebuild failed:', error);
storage.close();
console.error('\n❌ Documentation database rebuild failed:', error);
service.close();
process.exit(1);
}
}
// Run if called directly
if (require.main === module) {
rebuildDatabase().catch(error => {
rebuildDocumentationDatabase().catch(error => {
console.error('Fatal error:', error);
process.exit(1);
});
}
export { rebuildDatabase };
export { rebuildDocumentationDatabase };

171
src/services/node-documentation-service.ts
View File

@@ -4,7 +4,15 @@ import path from 'path';
import { promises as fs } from 'fs';
import { logger } from '../utils/logger';
import { NodeSourceExtractor } from '../utils/node-source-extractor';
import { DocumentationFetcher } from '../utils/documentation-fetcher';
import {
EnhancedDocumentationFetcher,
EnhancedNodeDocumentation,
OperationInfo,
ApiMethodMapping,
CodeExample,
TemplateInfo,
RelatedResource
} from '../utils/enhanced-documentation-fetcher';
import { ExampleGenerator } from '../utils/example-generator';
interface NodeInfo {
@@ -17,8 +25,15 @@ interface NodeInfo {
icon?: string;
sourceCode: string;
credentialCode?: string;
documentation?: string;
documentationMarkdown?: string;
documentationUrl?: string;
documentationTitle?: string;
operations?: OperationInfo[];
apiMethods?: ApiMethodMapping[];
documentationExamples?: CodeExample[];
templates?: TemplateInfo[];
relatedResources?: RelatedResource[];
requiredScopes?: string[];
exampleWorkflow?: any;
exampleParameters?: any;
propertiesSchema?: any;
@@ -44,7 +59,7 @@ interface SearchOptions {
export class NodeDocumentationService {
private db: Database.Database;
private extractor: NodeSourceExtractor;
private docsFetcher: DocumentationFetcher;
private docsFetcher: EnhancedDocumentationFetcher;
constructor(dbPath?: string) {
const databasePath = dbPath || process.env.NODE_DB_PATH || path.join(process.cwd(), 'data', 'nodes-v2.db');
@@ -57,7 +72,7 @@ export class NodeDocumentationService {
this.db = new Database(databasePath);
this.extractor = new NodeSourceExtractor();
this.docsFetcher = new DocumentationFetcher();
this.docsFetcher = new EnhancedDocumentationFetcher();
// Initialize database with new schema
this.initializeDatabase();
@@ -88,6 +103,15 @@ CREATE TABLE IF NOT EXISTS nodes (
-- Documentation
documentation_markdown TEXT,
documentation_url TEXT,
documentation_title TEXT,
-- Enhanced documentation fields (stored as JSON)
operations TEXT,
api_methods TEXT,
documentation_examples TEXT,
templates TEXT,
related_resources TEXT,
required_scopes TEXT,
-- Example usage
example_workflow TEXT,
@@ -182,14 +206,16 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
INSERT OR REPLACE INTO nodes (
node_type, name, display_name, description, category, subcategory, icon,
source_code, credential_code, code_hash, code_length,
documentation_markdown, documentation_url,
documentation_markdown, documentation_url, documentation_title,
operations, api_methods, documentation_examples, templates, related_resources, required_scopes,
example_workflow, example_parameters, properties_schema,
package_name, version, codex_data, aliases,
has_credentials, is_trigger, is_webhook
) VALUES (
@nodeType, @name, @displayName, @description, @category, @subcategory, @icon,
@sourceCode, @credentialCode, @hash, @codeLength,
@documentation, @documentationUrl,
@documentation, @documentationUrl, @documentationTitle,
@operations, @apiMethods, @documentationExamples, @templates, @relatedResources, @requiredScopes,
@exampleWorkflow, @exampleParameters, @propertiesSchema,
@packageName, @version, @codexData, @aliases,
@hasCredentials, @isTrigger, @isWebhook
@@ -208,8 +234,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
credentialCode: nodeInfo.credentialCode || null,
hash,
codeLength: nodeInfo.sourceCode.length,
documentation: nodeInfo.documentation || null,
documentation: nodeInfo.documentationMarkdown || null,
documentationUrl: nodeInfo.documentationUrl || null,
documentationTitle: nodeInfo.documentationTitle || null,
operations: nodeInfo.operations ? JSON.stringify(nodeInfo.operations) : null,
apiMethods: nodeInfo.apiMethods ? JSON.stringify(nodeInfo.apiMethods) : null,
documentationExamples: nodeInfo.documentationExamples ? JSON.stringify(nodeInfo.documentationExamples) : null,
templates: nodeInfo.templates ? JSON.stringify(nodeInfo.templates) : null,
relatedResources: nodeInfo.relatedResources ? JSON.stringify(nodeInfo.relatedResources) : null,
requiredScopes: nodeInfo.requiredScopes ? JSON.stringify(nodeInfo.requiredScopes) : null,
exampleWorkflow: nodeInfo.exampleWorkflow ? JSON.stringify(nodeInfo.exampleWorkflow) : null,
exampleParameters: nodeInfo.exampleParameters ? JSON.stringify(nodeInfo.exampleParameters) : null,
propertiesSchema: nodeInfo.propertiesSchema ? JSON.stringify(nodeInfo.propertiesSchema) : null,
@@ -346,13 +379,13 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
// Parse node definition to get metadata
const nodeDefinition = this.parseNodeDefinition(nodeData.sourceCode);
// Get documentation
const docs = await this.docsFetcher.getNodeDocumentation(nodeType);
// Get enhanced documentation
const enhancedDocs = await this.docsFetcher.getEnhancedNodeDocumentation(nodeType);
// Generate example
const example = ExampleGenerator.generateFromNodeDefinition(nodeDefinition);
// Prepare node info
// Prepare node info with enhanced documentation
const nodeInfo: NodeInfo = {
nodeType: nodeType,
name: node.name,
@@ -363,8 +396,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
icon: nodeDefinition.icon,
sourceCode: nodeData.sourceCode,
credentialCode: nodeData.credentialCode,
documentation: docs?.markdown,
documentationUrl: docs?.url,
documentationMarkdown: enhancedDocs?.markdown,
documentationUrl: enhancedDocs?.url,
documentationTitle: enhancedDocs?.title,
operations: enhancedDocs?.operations,
apiMethods: enhancedDocs?.apiMethods,
documentationExamples: enhancedDocs?.examples,
templates: enhancedDocs?.templates,
relatedResources: enhancedDocs?.relatedResources,
requiredScopes: enhancedDocs?.requiredScopes,
exampleWorkflow: example,
exampleParameters: example.nodes[0]?.parameters,
propertiesSchema: nodeDefinition.properties,
@@ -410,28 +450,88 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
* Parse node definition from source code
*/
private parseNodeDefinition(sourceCode: string): any {
try {
// Try to extract the description object from the source
const descMatch = sourceCode.match(/description\s*[:=]\s*({[\s\S]*?})\s*[,;]/);
if (descMatch) {
// Clean up the match and try to parse it
const descStr = descMatch[1]
.replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":') // Quote property names
.replace(/:\s*'([^']*)'/g, ': "$1"') // Convert single quotes to double
.replace(/,\s*}/g, '}'); // Remove trailing commas
return JSON.parse(descStr);
}
} catch (error) {
logger.debug('Failed to parse node definition:', error);
}
// Return minimal definition if parsing fails
return {
const result: any = {
displayName: '',
description: '',
properties: []
properties: [],
category: null,
subcategory: null,
icon: null,
version: null,
codex: null,
alias: null
};
try {
// Extract individual properties using specific patterns
// Display name
const displayNameMatch = sourceCode.match(/displayName\s*[:=]\s*['"`]([^'"`]+)['"`]/);
if (displayNameMatch) {
result.displayName = displayNameMatch[1];
}
// Description
const descriptionMatch = sourceCode.match(/description\s*[:=]\s*['"`]([^'"`]+)['"`]/);
if (descriptionMatch) {
result.description = descriptionMatch[1];
}
// Icon
const iconMatch = sourceCode.match(/icon\s*[:=]\s*['"`]([^'"`]+)['"`]/);
if (iconMatch) {
result.icon = iconMatch[1];
}
// Category/group
const groupMatch = sourceCode.match(/group\s*[:=]\s*\[['"`]([^'"`]+)['"`]\]/);
if (groupMatch) {
result.category = groupMatch[1];
}
// Version
const versionMatch = sourceCode.match(/version\s*[:=]\s*(\d+)/);
if (versionMatch) {
result.version = parseInt(versionMatch[1]);
}
// Subtitle
const subtitleMatch = sourceCode.match(/subtitle\s*[:=]\s*['"`]([^'"`]+)['"`]/);
if (subtitleMatch) {
result.subtitle = subtitleMatch[1];
}
// Try to extract properties array
const propsMatch = sourceCode.match(/properties\s*[:=]\s*(\[[\s\S]*?\])\s*[,}]/);
if (propsMatch) {
try {
// This is complex to parse from minified code, so we'll skip for now
result.properties = [];
} catch (e) {
// Ignore parsing errors
}
}
// Check if it's a trigger node
if (sourceCode.includes('implements.*ITrigger') ||
sourceCode.includes('polling:.*true') ||
sourceCode.includes('webhook:.*true') ||
result.displayName.toLowerCase().includes('trigger')) {
result.isTrigger = true;
}
// Check if it's a webhook node
if (sourceCode.includes('webhooks:') ||
sourceCode.includes('webhook:.*true') ||
result.displayName.toLowerCase().includes('webhook')) {
result.isWebhook = true;
}
} catch (error) {
logger.debug('Error parsing node definition:', error);
}
return result;
}
/**
@@ -448,8 +548,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
icon: row.icon,
sourceCode: row.source_code,
credentialCode: row.credential_code,
documentation: row.documentation_markdown,
documentationMarkdown: row.documentation_markdown,
documentationUrl: row.documentation_url,
documentationTitle: row.documentation_title,
operations: row.operations ? JSON.parse(row.operations) : null,
apiMethods: row.api_methods ? JSON.parse(row.api_methods) : null,
documentationExamples: row.documentation_examples ? JSON.parse(row.documentation_examples) : null,
templates: row.templates ? JSON.parse(row.templates) : null,
relatedResources: row.related_resources ? JSON.parse(row.related_resources) : null,
requiredScopes: row.required_scopes ? JSON.parse(row.required_scopes) : null,
exampleWorkflow: row.example_workflow ? JSON.parse(row.example_workflow) : null,
exampleParameters: row.example_parameters ? JSON.parse(row.example_parameters) : null,
propertiesSchema: row.properties_schema ? JSON.parse(row.properties_schema) : null,

274
src/services/node-storage-service.ts
View File

@@ -1,274 +0,0 @@
import { NodeSourceInfo } from '../utils/node-source-extractor';
import { logger } from '../utils/logger';
import * as crypto from 'crypto';
export interface StoredNode {
id: string;
nodeType: string;
name: string;
packageName: string;
displayName?: string;
description?: string;
codeHash: string;
codeLength: number;
sourceLocation: string;
hasCredentials: boolean;
extractedAt: Date;
updatedAt: Date;
sourceCode?: string;
credentialCode?: string;
packageInfo?: any;
metadata?: Record<string, any>;
}
export interface NodeSearchQuery {
query?: string;
packageName?: string;
nodeType?: string;
hasCredentials?: boolean;
limit?: number;
offset?: number;
}
export class NodeStorageService {
private nodes: Map<string, StoredNode> = new Map();
private nodesByPackage: Map<string, Set<string>> = new Map();
private searchIndex: Map<string, Set<string>> = new Map();
/**
* Store a node in the database
*/
async storeNode(nodeInfo: NodeSourceInfo): Promise<StoredNode> {
const codeHash = crypto.createHash('sha256').update(nodeInfo.sourceCode).digest('hex');
// Parse display name and description from source if possible
const displayName = this.extractDisplayName(nodeInfo.sourceCode);
const description = this.extractDescription(nodeInfo.sourceCode);
const storedNode: StoredNode = {
id: crypto.randomUUID(),
nodeType: nodeInfo.nodeType,
name: nodeInfo.nodeType.split('.').pop() || nodeInfo.nodeType,
packageName: nodeInfo.nodeType.split('.')[0] || 'unknown',
displayName,
description,
codeHash,
codeLength: nodeInfo.sourceCode.length,
sourceLocation: nodeInfo.location,
hasCredentials: !!nodeInfo.credentialCode,
extractedAt: new Date(),
updatedAt: new Date(),
sourceCode: nodeInfo.sourceCode,
credentialCode: nodeInfo.credentialCode,
packageInfo: nodeInfo.packageInfo,
};
// Store in memory (replace with real DB)
this.nodes.set(nodeInfo.nodeType, storedNode);
// Update package index
if (!this.nodesByPackage.has(storedNode.packageName)) {
this.nodesByPackage.set(storedNode.packageName, new Set());
}
this.nodesByPackage.get(storedNode.packageName)!.add(nodeInfo.nodeType);
// Update search index
this.updateSearchIndex(storedNode);
logger.info(`Stored node: ${nodeInfo.nodeType} (${codeHash.substring(0, 8)}...)`);
return storedNode;
}
/**
* Search for nodes
*/
async searchNodes(query: NodeSearchQuery): Promise<StoredNode[]> {
let results: StoredNode[] = [];
if (query.query) {
// Text search
const searchTerms = query.query.toLowerCase().split(' ');
const matchingNodeTypes = new Set<string>();
for (const term of searchTerms) {
const matches = this.searchIndex.get(term) || new Set();
matches.forEach(nodeType => matchingNodeTypes.add(nodeType));
}
results = Array.from(matchingNodeTypes)
.map(nodeType => this.nodes.get(nodeType)!)
.filter(Boolean);
} else {
// Get all nodes
results = Array.from(this.nodes.values());
}
// Apply filters
if (query.packageName) {
results = results.filter(node => node.packageName === query.packageName);
}
if (query.nodeType) {
results = results.filter(node => node.nodeType.includes(query.nodeType!));
}
if (query.hasCredentials !== undefined) {
results = results.filter(node => node.hasCredentials === query.hasCredentials);
}
// Apply pagination
const offset = query.offset || 0;
const limit = query.limit || 50;
return results.slice(offset, offset + limit);
}
/**
* Get node by type
*/
async getNode(nodeType: string): Promise<StoredNode | null> {
return this.nodes.get(nodeType) || null;
}
/**
* Get all packages
*/
async getPackages(): Promise<Array<{ name: string; nodeCount: number }>> {
return Array.from(this.nodesByPackage.entries()).map(([name, nodes]) => ({
name,
nodeCount: nodes.size,
}));
}
/**
* Bulk store nodes
*/
async bulkStoreNodes(nodeInfos: NodeSourceInfo[]): Promise<{
stored: number;
failed: number;
errors: Array<{ nodeType: string; error: string }>;
}> {
const results = {
stored: 0,
failed: 0,
errors: [] as Array<{ nodeType: string; error: string }>,
};
for (const nodeInfo of nodeInfos) {
try {
await this.storeNode(nodeInfo);
results.stored++;
} catch (error) {
results.failed++;
results.errors.push({
nodeType: nodeInfo.nodeType,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
}
return results;
}
/**
* Generate statistics
*/
async getStatistics(): Promise<{
totalNodes: number;
totalPackages: number;
totalCodeSize: number;
nodesWithCredentials: number;
averageNodeSize: number;
packageDistribution: Array<{ package: string; count: number }>;
}> {
const nodes = Array.from(this.nodes.values());
const totalCodeSize = nodes.reduce((sum, node) => sum + node.codeLength, 0);
const nodesWithCredentials = nodes.filter(node => node.hasCredentials).length;
const packageDistribution = Array.from(this.nodesByPackage.entries())
.map(([pkg, nodeSet]) => ({ package: pkg, count: nodeSet.size }))
.sort((a, b) => b.count - a.count);
return {
totalNodes: nodes.length,
totalPackages: this.nodesByPackage.size,
totalCodeSize,
nodesWithCredentials,
averageNodeSize: nodes.length > 0 ? Math.round(totalCodeSize / nodes.length) : 0,
packageDistribution,
};
}
/**
* Extract display name from source code
*/
private extractDisplayName(sourceCode: string): string | undefined {
const match = sourceCode.match(/displayName:\s*["'`]([^"'`]+)["'`]/);
return match ? match[1] : undefined;
}
/**
* Extract description from source code
*/
private extractDescription(sourceCode: string): string | undefined {
const match = sourceCode.match(/description:\s*["'`]([^"'`]+)["'`]/);
return match ? match[1] : undefined;
}
/**
* Update search index
*/
private updateSearchIndex(node: StoredNode): void {
// Index by name parts
const nameParts = node.name.toLowerCase().split(/(?=[A-Z])|[._-]/).filter(Boolean);
for (const part of nameParts) {
if (!this.searchIndex.has(part)) {
this.searchIndex.set(part, new Set());
}
this.searchIndex.get(part)!.add(node.nodeType);
}
// Index by display name
if (node.displayName) {
const displayParts = node.displayName.toLowerCase().split(/\s+/);
for (const part of displayParts) {
if (!this.searchIndex.has(part)) {
this.searchIndex.set(part, new Set());
}
this.searchIndex.get(part)!.add(node.nodeType);
}
}
// Index by package name
const pkgParts = node.packageName.toLowerCase().split(/[.-]/);
for (const part of pkgParts) {
if (!this.searchIndex.has(part)) {
this.searchIndex.set(part, new Set());
}
this.searchIndex.get(part)!.add(node.nodeType);
}
}
/**
* Export all nodes for database import
*/
async exportForDatabase(): Promise<{
nodes: StoredNode[];
metadata: {
exportedAt: Date;
totalNodes: number;
totalPackages: number;
};
}> {
const nodes = Array.from(this.nodes.values());
return {
nodes,
metadata: {
exportedAt: new Date(),
totalNodes: nodes.length,
totalPackages: this.nodesByPackage.size,
},
};
}
}

410
src/services/sqlite-storage-service.ts
View File

@@ -1,410 +0,0 @@
import Database from 'better-sqlite3';
import * as path from 'path';
import * as fs from 'fs';
import * as crypto from 'crypto';
import { NodeSourceInfo } from '../utils/node-source-extractor';
import { StoredNode, NodeSearchQuery } from './node-storage-service';
import { logger } from '../utils/logger';
export class SQLiteStorageService {
private db: Database.Database;
private readonly dbPath: string;
constructor(dbPath?: string) {
this.dbPath = dbPath || process.env.NODE_DB_PATH || path.join(process.cwd(), 'data', 'nodes.db');
// Ensure data directory exists
const dataDir = path.dirname(this.dbPath);
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
}
this.db = new Database(this.dbPath, {
verbose: process.env.NODE_ENV === 'development' ? (msg: unknown) => logger.debug(String(msg)) : undefined
});
// Enable WAL mode for better performance
this.db.pragma('journal_mode = WAL');
this.initializeDatabase();
}
/**
* Initialize database with schema
*/
private initializeDatabase(): void {
try {
const schema = `
-- Main nodes table
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_type TEXT UNIQUE NOT NULL,
name TEXT NOT NULL,
package_name TEXT NOT NULL,
display_name TEXT,
description TEXT,
code_hash TEXT NOT NULL,
code_length INTEGER NOT NULL,
source_location TEXT NOT NULL,
source_code TEXT NOT NULL,
credential_code TEXT,
package_info TEXT, -- JSON
has_credentials INTEGER DEFAULT 0,
extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Indexes for performance
CREATE INDEX IF NOT EXISTS idx_nodes_package_name ON nodes(package_name);
CREATE INDEX IF NOT EXISTS idx_nodes_code_hash ON nodes(code_hash);
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
-- Full Text Search virtual table for node search
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
node_type,
name,
display_name,
description,
package_name,
content=nodes,
content_rowid=id
);
-- Triggers to keep FTS in sync
CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes
BEGIN
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
END;
CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes
BEGIN
DELETE FROM nodes_fts WHERE rowid = old.id;
END;
CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes
BEGIN
DELETE FROM nodes_fts WHERE rowid = old.id;
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
END;
-- Statistics table for metadata
CREATE TABLE IF NOT EXISTS extraction_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT,
total_nodes INTEGER NOT NULL,
total_packages INTEGER NOT NULL,
total_code_size INTEGER NOT NULL,
nodes_with_credentials INTEGER NOT NULL,
extraction_date DATETIME DEFAULT CURRENT_TIMESTAMP
);
`;
this.db.exec(schema);
logger.info('Database initialized successfully');
} catch (error) {
logger.error('Failed to initialize database:', error);
throw error;
}
}
/**
* Store a node in the database
*/
async storeNode(nodeInfo: NodeSourceInfo): Promise<StoredNode> {
const codeHash = crypto.createHash('sha256').update(nodeInfo.sourceCode).digest('hex');
// Parse display name and description from source
const displayName = this.extractDisplayName(nodeInfo.sourceCode);
const description = this.extractDescription(nodeInfo.sourceCode);
const stmt = this.db.prepare(`
INSERT OR REPLACE INTO nodes (
node_type, name, package_name, display_name, description,
code_hash, code_length, source_location, source_code,
credential_code, package_info, has_credentials,
updated_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP
)
`);
const name = nodeInfo.nodeType.split('.').pop() || nodeInfo.nodeType;
const packageName = nodeInfo.nodeType.split('.')[0] || 'unknown';
const result = stmt.run(
nodeInfo.nodeType,
name,
packageName,
displayName || null,
description || null,
codeHash,
nodeInfo.sourceCode.length,
nodeInfo.location,
nodeInfo.sourceCode,
nodeInfo.credentialCode || null,
nodeInfo.packageInfo ? JSON.stringify(nodeInfo.packageInfo) : null,
nodeInfo.credentialCode ? 1 : 0
);
logger.info(`Stored node: ${nodeInfo.nodeType} (${codeHash.substring(0, 8)}...)`);
return {
id: String(result.lastInsertRowid),
nodeType: nodeInfo.nodeType,
name,
packageName,
displayName,
description,
codeHash,
codeLength: nodeInfo.sourceCode.length,
sourceLocation: nodeInfo.location,
hasCredentials: !!nodeInfo.credentialCode,
extractedAt: new Date(),
updatedAt: new Date(),
sourceCode: nodeInfo.sourceCode,
credentialCode: nodeInfo.credentialCode,
packageInfo: nodeInfo.packageInfo
};
}
/**
* Search for nodes using FTS
*/
async searchNodes(query: NodeSearchQuery): Promise<StoredNode[]> {
let sql = `
SELECT DISTINCT n.*
FROM nodes n
`;
const params: any[] = [];
const conditions: string[] = [];
if (query.query) {
// Use FTS for text search
sql += ` JOIN nodes_fts fts ON n.id = fts.rowid`;
conditions.push(`nodes_fts MATCH ?`);
// Convert search query to FTS syntax (prefix search)
const ftsQuery = query.query.split(' ')
.map(term => `${term}*`)
.join(' ');
params.push(ftsQuery);
}
if (query.packageName) {
conditions.push(`n.package_name = ?`);
params.push(query.packageName);
}
if (query.nodeType) {
conditions.push(`n.node_type LIKE ?`);
params.push(`%${query.nodeType}%`);
}
if (query.hasCredentials !== undefined) {
conditions.push(`n.has_credentials = ?`);
params.push(query.hasCredentials ? 1 : 0);
}
if (conditions.length > 0) {
sql += ` WHERE ${conditions.join(' AND ')}`;
}
sql += ` ORDER BY n.name`;
if (query.limit) {
sql += ` LIMIT ?`;
params.push(query.limit);
if (query.offset) {
sql += ` OFFSET ?`;
params.push(query.offset);
}
}
const stmt = this.db.prepare(sql);
const rows = stmt.all(...params);
return rows.map(row => this.rowToStoredNode(row));
}
/**
* Get node by type
*/
async getNode(nodeType: string): Promise<StoredNode | null> {
const stmt = this.db.prepare(`
SELECT * FROM nodes WHERE node_type = ?
`);
const row = stmt.get(nodeType);
return row ? this.rowToStoredNode(row) : null;
}
/**
* Get all packages
*/
async getPackages(): Promise<Array<{ name: string; nodeCount: number }>> {
const stmt = this.db.prepare(`
SELECT package_name as name, COUNT(*) as nodeCount
FROM nodes
GROUP BY package_name
ORDER BY nodeCount DESC
`);
return stmt.all() as Array<{ name: string; nodeCount: number }>;
}
/**
* Bulk store nodes (used for database rebuild)
*/
async bulkStoreNodes(nodeInfos: NodeSourceInfo[]): Promise<{
stored: number;
failed: number;
errors: Array<{ nodeType: string; error: string }>;
}> {
const results = {
stored: 0,
failed: 0,
errors: [] as Array<{ nodeType: string; error: string }>
};
// Use transaction for bulk insert
const insertMany = this.db.transaction((nodes: NodeSourceInfo[]) => {
for (const nodeInfo of nodes) {
try {
this.storeNode(nodeInfo);
results.stored++;
} catch (error) {
results.failed++;
results.errors.push({
nodeType: nodeInfo.nodeType,
error: error instanceof Error ? error.message : 'Unknown error'
});
}
}
});
insertMany(nodeInfos);
return results;
}
/**
* Get statistics
*/
async getStatistics(): Promise<{
totalNodes: number;
totalPackages: number;
totalCodeSize: number;
nodesWithCredentials: number;
averageNodeSize: number;
packageDistribution: Array<{ package: string; count: number }>;
}> {
const stats = this.db.prepare(`
SELECT
COUNT(*) as totalNodes,
COUNT(DISTINCT package_name) as totalPackages,
SUM(code_length) as totalCodeSize,
SUM(has_credentials) as nodesWithCredentials
FROM nodes
`).get() as any;
const packageDist = this.db.prepare(`
SELECT package_name as package, COUNT(*) as count
FROM nodes
GROUP BY package_name
ORDER BY count DESC
`).all() as Array<{ package: string; count: number }>;
return {
totalNodes: stats.totalNodes || 0,
totalPackages: stats.totalPackages || 0,
totalCodeSize: stats.totalCodeSize || 0,
nodesWithCredentials: stats.nodesWithCredentials || 0,
averageNodeSize: stats.totalNodes > 0 ? Math.round(stats.totalCodeSize / stats.totalNodes) : 0,
packageDistribution: packageDist
};
}
/**
* Rebuild entire database
*/
async rebuildDatabase(): Promise<void> {
logger.info('Starting database rebuild...');
// Clear existing data
this.db.exec('DELETE FROM nodes');
this.db.exec('DELETE FROM extraction_stats');
logger.info('Database cleared for rebuild');
}
/**
* Save extraction statistics
*/
async saveExtractionStats(stats: {
totalNodes: number;
totalPackages: number;
totalCodeSize: number;
nodesWithCredentials: number;
}): Promise<void> {
const stmt = this.db.prepare(`
INSERT INTO extraction_stats (
total_nodes, total_packages, total_code_size, nodes_with_credentials
) VALUES (?, ?, ?, ?)
`);
stmt.run(
stats.totalNodes,
stats.totalPackages,
stats.totalCodeSize,
stats.nodesWithCredentials
);
}
/**
* Close database connection
*/
close(): void {
this.db.close();
}
/**
* Convert database row to StoredNode
*/
private rowToStoredNode(row: any): StoredNode {
return {
id: String(row.id),
nodeType: row.node_type,
name: row.name,
packageName: row.package_name,
displayName: row.display_name,
description: row.description,
codeHash: row.code_hash,
codeLength: row.code_length,
sourceLocation: row.source_location,
hasCredentials: row.has_credentials === 1,
extractedAt: new Date(row.extracted_at),
updatedAt: new Date(row.updated_at),
sourceCode: row.source_code,
credentialCode: row.credential_code,
packageInfo: row.package_info ? JSON.parse(row.package_info) : undefined
};
}
/**
* Extract display name from source code
*/
private extractDisplayName(sourceCode: string): string | undefined {
const match = sourceCode.match(/displayName:\s*["'`]([^"'`]+)["'`]/);
return match ? match[1] : undefined;
}
/**
* Extract description from source code
*/
private extractDescription(sourceCode: string): string | undefined {
const match = sourceCode.match(/description:\s*["'`]([^"'`]+)["'`]/);
return match ? match[1] : undefined;
}
}

49
src/utils/auth-middleware.ts
View File

@@ -1,49 +0,0 @@
import { Request, Response, NextFunction } from 'express';
import { logger } from './logger';
/**
* Express middleware for authenticating requests with Bearer tokens
*/
export function authenticateRequest(authToken?: string) {
return (req: Request, res: Response, next: NextFunction): void => {
if (!authToken) {
// No auth required
return next();
}
const authHeader = req.headers['authorization'];
if (!authHeader) {
logger.warn('Missing authorization header', {
ip: req.ip,
path: req.path,
});
res.status(401).json({
error: 'Unauthorized',
message: 'Missing authorization header',
});
return;
}
// Support both "Bearer TOKEN" and just "TOKEN" formats
const providedToken = authHeader.startsWith('Bearer ')
? authHeader.substring(7)
: authHeader;
if (providedToken !== authToken) {
logger.warn('Invalid authentication token', {
ip: req.ip,
path: req.path,
});
res.status(401).json({
error: 'Unauthorized',
message: 'Invalid authentication token',
});
return;
}
next();
};
}

243
src/utils/documentation-fetcher.ts
View File

@@ -1,241 +1,2 @@
import { promises as fs } from 'fs';
import path from 'path';
import { logger } from './logger';
import { execSync } from 'child_process';
interface NodeDocumentation {
markdown: string;
url: string;
examples?: any[];
}
export class DocumentationFetcher {
private docsPath: string;
private docsRepoUrl = 'https://github.com/n8n-io/n8n-docs.git';
private cloned = false;
constructor(docsPath?: string) {
this.docsPath = docsPath || path.join(process.cwd(), 'temp', 'n8n-docs');
}
/**
* Clone or update the n8n-docs repository
*/
async ensureDocsRepository(): Promise<void> {
try {
// Check if directory exists
const exists = await fs.access(this.docsPath).then(() => true).catch(() => false);
if (!exists) {
logger.info('Cloning n8n-docs repository...');
await fs.mkdir(path.dirname(this.docsPath), { recursive: true });
execSync(`git clone --depth 1 ${this.docsRepoUrl} ${this.docsPath}`, {
stdio: 'pipe'
});
logger.info('n8n-docs repository cloned successfully');
} else {
logger.info('Updating n8n-docs repository...');
execSync('git pull --ff-only', {
cwd: this.docsPath,
stdio: 'pipe'
});
logger.info('n8n-docs repository updated');
}
this.cloned = true;
} catch (error) {
logger.error('Failed to clone/update n8n-docs repository:', error);
throw error;
}
}
/**
* Get documentation for a specific node
*/
async getNodeDocumentation(nodeType: string): Promise<NodeDocumentation | null> {
if (!this.cloned) {
await this.ensureDocsRepository();
}
try {
// Convert node type to documentation path
// e.g., "n8n-nodes-base.if" -> "if"
const nodeName = this.extractNodeName(nodeType);
// Common documentation paths to check
const possiblePaths = [
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'core-nodes', `${nodeName}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'app-nodes', `${nodeName}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'trigger-nodes', `${nodeName}.md`),
path.join(this.docsPath, 'docs', 'code-examples', 'expressions', `${nodeName}.md`),
// Generic search in docs folder
path.join(this.docsPath, 'docs', '**', `${nodeName}.md`)
];
for (const docPath of possiblePaths) {
try {
const content = await fs.readFile(docPath, 'utf-8');
const url = this.generateDocUrl(docPath);
return {
markdown: content,
url,
examples: this.extractExamples(content)
};
} catch (error) {
// Continue to next path
continue;
}
}
// If no exact match, try to find by searching
const foundPath = await this.searchForNodeDoc(nodeName);
if (foundPath) {
const content = await fs.readFile(foundPath, 'utf-8');
return {
markdown: content,
url: this.generateDocUrl(foundPath),
examples: this.extractExamples(content)
};
}
logger.warn(`No documentation found for node: ${nodeType}`);
return null;
} catch (error) {
logger.error(`Failed to get documentation for ${nodeType}:`, error);
return null;
}
}
/**
* Extract node name from node type
*/
private extractNodeName(nodeType: string): string {
// Handle different node type formats
// "n8n-nodes-base.if" -> "if"
// "@n8n/n8n-nodes-langchain.Agent" -> "agent"
const parts = nodeType.split('.');
const name = parts[parts.length - 1];
return name.toLowerCase();
}
/**
* Search for node documentation file
*/
private async searchForNodeDoc(nodeName: string): Promise<string | null> {
try {
const result = execSync(
`find ${this.docsPath}/docs -name "*.md" -type f | grep -i "${nodeName}" | head -1`,
{ encoding: 'utf-8', stdio: 'pipe' }
).trim();
return result || null;
} catch (error) {
return null;
}
}
/**
* Generate documentation URL from file path
*/
private generateDocUrl(filePath: string): string {
const relativePath = path.relative(this.docsPath, filePath);
const urlPath = relativePath
.replace(/^docs\//, '')
.replace(/\.md$/, '')
.replace(/\\/g, '/');
return `https://docs.n8n.io/${urlPath}`;
}
/**
* Extract code examples from markdown content
*/
private extractExamples(markdown: string): any[] {
const examples: any[] = [];
// Extract JSON code blocks
const jsonCodeBlockRegex = /```json\n([\s\S]*?)```/g;
let match;
while ((match = jsonCodeBlockRegex.exec(markdown)) !== null) {
try {
const json = JSON.parse(match[1]);
examples.push(json);
} catch (error) {
// Not valid JSON, skip
}
}
// Extract workflow examples
const workflowExampleRegex = /## Example.*?\n([\s\S]*?)(?=\n##|\n#|$)/gi;
while ((match = workflowExampleRegex.exec(markdown)) !== null) {
const exampleText = match[1];
// Try to find JSON in the example section
const jsonMatch = exampleText.match(/```json\n([\s\S]*?)```/);
if (jsonMatch) {
try {
const json = JSON.parse(jsonMatch[1]);
examples.push(json);
} catch (error) {
// Not valid JSON
}
}
}
return examples;
}
/**
* Get all available documentation files
*/
async getAllDocumentationFiles(): Promise<Map<string, string>> {
if (!this.cloned) {
await this.ensureDocsRepository();
}
const docMap = new Map<string, string>();
try {
const findDocs = execSync(
`find ${this.docsPath}/docs -name "*.md" -type f | grep -E "(core-nodes|app-nodes|trigger-nodes)/"`,
{ encoding: 'utf-8', stdio: 'pipe' }
).trim().split('\n');
for (const docPath of findDocs) {
if (!docPath) continue;
const filename = path.basename(docPath, '.md');
const content = await fs.readFile(docPath, 'utf-8');
// Try to extract the node type from the content
const nodeTypeMatch = content.match(/node[_-]?type[:\s]+["']?([^"'\s]+)["']?/i);
if (nodeTypeMatch) {
docMap.set(nodeTypeMatch[1], docPath);
} else {
// Use filename as fallback
docMap.set(filename, docPath);
}
}
logger.info(`Found ${docMap.size} documentation files`);
return docMap;
} catch (error) {
logger.error('Failed to get documentation files:', error);
return docMap;
}
}
/**
* Clean up cloned repository
*/
async cleanup(): Promise<void> {
try {
await fs.rm(this.docsPath, { recursive: true, force: true });
this.cloned = false;
logger.info('Cleaned up documentation repository');
} catch (error) {
logger.error('Failed to cleanup docs repository:', error);
}
}
}
// Re-export everything from enhanced-documentation-fetcher
export * from './enhanced-documentation-fetcher';

621
src/utils/enhanced-documentation-fetcher.ts Normal file
View File

@@ -0,0 +1,621 @@
import { promises as fs } from 'fs';
import path from 'path';
import { logger } from './logger';
import { execSync } from 'child_process';
// Enhanced documentation structure with rich content
export interface EnhancedNodeDocumentation {
markdown: string;
url: string;
title?: string;
description?: string;
operations?: OperationInfo[];
apiMethods?: ApiMethodMapping[];
examples?: CodeExample[];
templates?: TemplateInfo[];
relatedResources?: RelatedResource[];
requiredScopes?: string[];
metadata?: DocumentationMetadata;
}
export interface OperationInfo {
resource: string;
operation: string;
description: string;
subOperations?: string[];
}
export interface ApiMethodMapping {
resource: string;
operation: string;
apiMethod: string;
apiUrl: string;
}
export interface CodeExample {
title?: string;
description?: string;
type: 'json' | 'javascript' | 'yaml' | 'text';
code: string;
language?: string;
}
export interface TemplateInfo {
name: string;
description?: string;
url?: string;
}
export interface RelatedResource {
title: string;
url: string;
type: 'documentation' | 'api' | 'tutorial' | 'external';
}
export interface DocumentationMetadata {
contentType?: string[];
priority?: string;
tags?: string[];
lastUpdated?: Date;
}
export class EnhancedDocumentationFetcher {
private docsPath: string;
private docsRepoUrl = 'https://github.com/n8n-io/n8n-docs.git';
private cloned = false;
constructor(docsPath?: string) {
this.docsPath = docsPath || path.join(process.cwd(), 'temp', 'n8n-docs');
}
/**
* Clone or update the n8n-docs repository
*/
async ensureDocsRepository(): Promise<void> {
try {
const exists = await fs.access(this.docsPath).then(() => true).catch(() => false);
if (!exists) {
logger.info('Cloning n8n-docs repository...');
await fs.mkdir(path.dirname(this.docsPath), { recursive: true });
execSync(`git clone --depth 1 ${this.docsRepoUrl} ${this.docsPath}`, {
stdio: 'pipe'
});
logger.info('n8n-docs repository cloned successfully');
} else {
logger.info('Updating n8n-docs repository...');
execSync('git pull --ff-only', {
cwd: this.docsPath,
stdio: 'pipe'
});
logger.info('n8n-docs repository updated');
}
this.cloned = true;
} catch (error) {
logger.error('Failed to clone/update n8n-docs repository:', error);
throw error;
}
}
/**
* Get enhanced documentation for a specific node
*/
async getEnhancedNodeDocumentation(nodeType: string): Promise<EnhancedNodeDocumentation | null> {
if (!this.cloned) {
await this.ensureDocsRepository();
}
try {
const nodeName = this.extractNodeName(nodeType);
// Common documentation paths to check
const possiblePaths = [
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'app-nodes', `${nodeType}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'core-nodes', `${nodeType}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'trigger-nodes', `${nodeType}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'core-nodes', `${nodeName}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'app-nodes', `${nodeName}.md`),
path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'trigger-nodes', `${nodeName}.md`),
];
for (const docPath of possiblePaths) {
try {
const content = await fs.readFile(docPath, 'utf-8');
logger.debug(`Checking doc path: ${docPath}`);
// Skip credential documentation files
if (this.isCredentialDoc(docPath, content)) {
logger.debug(`Skipping credential doc: ${docPath}`);
continue;
}
logger.info(`Found documentation for ${nodeType} at: ${docPath}`);
return this.parseEnhancedDocumentation(content, docPath);
} catch (error) {
// File doesn't exist, continue
continue;
}
}
// If no exact match, try to find by searching
logger.debug(`No exact match found, searching for ${nodeType}...`);
const foundPath = await this.searchForNodeDoc(nodeType);
if (foundPath) {
logger.info(`Found documentation via search at: ${foundPath}`);
const content = await fs.readFile(foundPath, 'utf-8');
if (!this.isCredentialDoc(foundPath, content)) {
return this.parseEnhancedDocumentation(content, foundPath);
}
}
logger.warn(`No documentation found for node: ${nodeType}`);
return null;
} catch (error) {
logger.error(`Failed to get documentation for ${nodeType}:`, error);
return null;
}
}
/**
* Parse markdown content into enhanced documentation structure
*/
private parseEnhancedDocumentation(markdown: string, filePath: string): EnhancedNodeDocumentation {
const doc: EnhancedNodeDocumentation = {
markdown,
url: this.generateDocUrl(filePath),
};
// Extract frontmatter metadata
const metadata = this.extractFrontmatter(markdown);
if (metadata) {
doc.metadata = metadata;
doc.title = metadata.title;
doc.description = metadata.description;
}
// Extract title and description from content if not in frontmatter
if (!doc.title) {
doc.title = this.extractTitle(markdown);
}
if (!doc.description) {
doc.description = this.extractDescription(markdown);
}
// Extract operations
doc.operations = this.extractOperations(markdown);
// Extract API method mappings
doc.apiMethods = this.extractApiMethods(markdown);
// Extract code examples
doc.examples = this.extractCodeExamples(markdown);
// Extract templates
doc.templates = this.extractTemplates(markdown);
// Extract related resources
doc.relatedResources = this.extractRelatedResources(markdown);
// Extract required scopes
doc.requiredScopes = this.extractRequiredScopes(markdown);
return doc;
}
/**
* Extract frontmatter metadata
*/
private extractFrontmatter(markdown: string): any {
const frontmatterMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
if (!frontmatterMatch) return null;
const frontmatter: any = {};
const lines = frontmatterMatch[1].split('\n');
for (const line of lines) {
if (line.includes(':')) {
const [key, ...valueParts] = line.split(':');
const value = valueParts.join(':').trim();
// Parse arrays
if (value.startsWith('[') && value.endsWith(']')) {
frontmatter[key.trim()] = value
.slice(1, -1)
.split(',')
.map(v => v.trim());
} else {
frontmatter[key.trim()] = value;
}
}
}
return frontmatter;
}
/**
* Extract title from markdown
*/
private extractTitle(markdown: string): string | undefined {
const match = markdown.match(/^#\s+(.+)$/m);
return match ? match[1].trim() : undefined;
}
/**
* Extract description from markdown
*/
private extractDescription(markdown: string): string | undefined {
// Remove frontmatter
const content = markdown.replace(/^---[\s\S]*?---\n/, '');
// Find first paragraph after title
const lines = content.split('\n');
let foundTitle = false;
let description = '';
for (const line of lines) {
if (line.startsWith('#')) {
foundTitle = true;
continue;
}
if (foundTitle && line.trim() && !line.startsWith('#') && !line.startsWith('*') && !line.startsWith('-')) {
description = line.trim();
break;
}
}
return description || undefined;
}
/**
* Extract operations from markdown
*/
private extractOperations(markdown: string): OperationInfo[] {
const operations: OperationInfo[] = [];
// Find operations section
const operationsMatch = markdown.match(/##\s+Operations\s*\n([\s\S]*?)(?=\n##|\n#|$)/i);
if (!operationsMatch) return operations;
const operationsText = operationsMatch[1];
// Parse operation structure - handle nested bullet points
let currentResource: string | null = null;
const lines = operationsText.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
// Skip empty lines
if (!trimmedLine) continue;
// Resource level - non-indented bullet with bold text (e.g., "* **Channel**")
if (line.match(/^\*\s+\*\*[^*]+\*\*\s*$/) && !line.match(/^\s+/)) {
const match = trimmedLine.match(/^\*\s+\*\*([^*]+)\*\*/);
if (match) {
currentResource = match[1].trim();
}
continue;
}
// Skip if we don't have a current resource
if (!currentResource) continue;
// Operation level - indented bullets (any whitespace + *)
if (line.match(/^\s+\*\s+/) && currentResource) {
// Extract operation name and description
const operationMatch = trimmedLine.match(/^\*\s+\*\*([^*]+)\*\*(.*)$/);
if (operationMatch) {
const operation = operationMatch[1].trim();
let description = operationMatch[2].trim();
// Clean up description
description = description.replace(/^:\s*/, '').replace(/\.$/, '').trim();
operations.push({
resource: currentResource,
operation,
description: description || operation,
});
} else {
// Handle operations without bold formatting or with different format
const simpleMatch = trimmedLine.match(/^\*\s+(.+)$/);
if (simpleMatch) {
const text = simpleMatch[1].trim();
// Split by colon to separate operation from description
const colonIndex = text.indexOf(':');
if (colonIndex > 0) {
operations.push({
resource: currentResource,
operation: text.substring(0, colonIndex).trim(),
description: text.substring(colonIndex + 1).trim() || text,
});
} else {
operations.push({
resource: currentResource,
operation: text,
description: text,
});
}
}
}
}
}
return operations;
}
/**
* Extract API method mappings from markdown tables
*/
private extractApiMethods(markdown: string): ApiMethodMapping[] {
const apiMethods: ApiMethodMapping[] = [];
// Find API method tables
const tableRegex = /\|.*Resource.*\|.*Operation.*\|.*(?:Slack API method|API method|Method).*\|[\s\S]*?\n(?=\n[^|]|$)/gi;
const tables = markdown.match(tableRegex);
if (!tables) return apiMethods;
for (const table of tables) {
const rows = table.split('\n').filter(row => row.trim() && !row.includes('---'));
// Skip header row
for (let i = 1; i < rows.length; i++) {
const cells = rows[i].split('|').map(cell => cell.trim()).filter(Boolean);
if (cells.length >= 3) {
const resource = cells[0];
const operation = cells[1];
const apiMethodCell = cells[2];
// Extract API method and URL from markdown link
const linkMatch = apiMethodCell.match(/\[([^\]]+)\]\(([^)]+)\)/);
if (linkMatch) {
apiMethods.push({
resource,
operation,
apiMethod: linkMatch[1],
apiUrl: linkMatch[2],
});
} else {
apiMethods.push({
resource,
operation,
apiMethod: apiMethodCell,
apiUrl: '',
});
}
}
}
}
return apiMethods;
}
/**
* Extract code examples from markdown
*/
private extractCodeExamples(markdown: string): CodeExample[] {
const examples: CodeExample[] = [];
// Extract all code blocks with language
const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
let match;
while ((match = codeBlockRegex.exec(markdown)) !== null) {
const language = match[1] || 'text';
const code = match[2].trim();
// Look for title or description before the code block
const beforeCodeIndex = match.index;
const beforeText = markdown.substring(Math.max(0, beforeCodeIndex - 200), beforeCodeIndex);
const titleMatch = beforeText.match(/(?:###|####)\s+(.+)$/m);
const example: CodeExample = {
type: this.mapLanguageToType(language),
language,
code,
};
if (titleMatch) {
example.title = titleMatch[1].trim();
}
// Try to parse JSON examples
if (language === 'json') {
try {
JSON.parse(code);
examples.push(example);
} catch (e) {
// Skip invalid JSON
}
} else {
examples.push(example);
}
}
return examples;
}
/**
* Extract template information
*/
private extractTemplates(markdown: string): TemplateInfo[] {
const templates: TemplateInfo[] = [];
// Look for template widget
const templateWidgetMatch = markdown.match(/\[\[\s*templatesWidget\s*\(\s*[^,]+,\s*'([^']+)'\s*\)\s*\]\]/);
if (templateWidgetMatch) {
templates.push({
name: templateWidgetMatch[1],
description: `Templates for ${templateWidgetMatch[1]}`,
});
}
return templates;
}
/**
* Extract related resources
*/
private extractRelatedResources(markdown: string): RelatedResource[] {
const resources: RelatedResource[] = [];
// Find related resources section
const relatedMatch = markdown.match(/##\s+(?:Related resources|Related|Resources)\s*\n([\s\S]*?)(?=\n##|\n#|$)/i);
if (!relatedMatch) return resources;
const relatedText = relatedMatch[1];
// Extract links
const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
let match;
while ((match = linkRegex.exec(relatedText)) !== null) {
const title = match[1];
const url = match[2];
// Determine resource type
let type: RelatedResource['type'] = 'external';
if (url.includes('docs.n8n.io') || url.startsWith('/')) {
type = 'documentation';
} else if (url.includes('api.')) {
type = 'api';
}
resources.push({ title, url, type });
}
return resources;
}
/**
* Extract required scopes
*/
private extractRequiredScopes(markdown: string): string[] {
const scopes: string[] = [];
// Find required scopes section
const scopesMatch = markdown.match(/##\s+(?:Required scopes|Scopes)\s*\n([\s\S]*?)(?=\n##|\n#|$)/i);
if (!scopesMatch) return scopes;
const scopesText = scopesMatch[1];
// Extract scope patterns (common formats)
const scopeRegex = /`([a-z:._-]+)`/gi;
let match;
while ((match = scopeRegex.exec(scopesText)) !== null) {
const scope = match[1];
if (scope.includes(':') || scope.includes('.')) {
scopes.push(scope);
}
}
return [...new Set(scopes)]; // Remove duplicates
}
/**
* Map language to code example type
*/
private mapLanguageToType(language: string): CodeExample['type'] {
switch (language.toLowerCase()) {
case 'json':
return 'json';
case 'js':
case 'javascript':
case 'typescript':
case 'ts':
return 'javascript';
case 'yaml':
case 'yml':
return 'yaml';
default:
return 'text';
}
}
/**
* Check if this is a credential documentation
*/
private isCredentialDoc(filePath: string, content: string): boolean {
return filePath.includes('/credentials/') ||
(content.includes('title: ') &&
content.includes(' credentials') &&
!content.includes(' node documentation'));
}
/**
* Extract node name from node type
*/
private extractNodeName(nodeType: string): string {
const parts = nodeType.split('.');
const name = parts[parts.length - 1];
return name.toLowerCase();
}
/**
* Search for node documentation file
*/
private async searchForNodeDoc(nodeType: string): Promise<string | null> {
try {
// First try exact match with nodeType
let result = execSync(
`find ${this.docsPath}/docs/integrations/builtin -name "${nodeType}.md" -type f | grep -v credentials | head -1`,
{ encoding: 'utf-8', stdio: 'pipe' }
).trim();
if (result) return result;
// Try lowercase nodeType
const lowerNodeType = nodeType.toLowerCase();
result = execSync(
`find ${this.docsPath}/docs/integrations/builtin -name "${lowerNodeType}.md" -type f | grep -v credentials | head -1`,
{ encoding: 'utf-8', stdio: 'pipe' }
).trim();
if (result) return result;
// Try node name pattern but exclude trigger nodes
const nodeName = this.extractNodeName(nodeType);
result = execSync(
`find ${this.docsPath}/docs/integrations/builtin -name "*${nodeName}.md" -type f | grep -v credentials | grep -v trigger | head -1`,
{ encoding: 'utf-8', stdio: 'pipe' }
).trim();
return result || null;
} catch (error) {
return null;
}
}
/**
* Generate documentation URL from file path
*/
private generateDocUrl(filePath: string): string {
const relativePath = path.relative(this.docsPath, filePath);
const urlPath = relativePath
.replace(/^docs\//, '')
.replace(/\.md$/, '')
.replace(/\\/g, '/');
return `https://docs.n8n.io/${urlPath}`;
}
/**
* Clean up cloned repository
*/
async cleanup(): Promise<void> {
try {
await fs.rm(this.docsPath, { recursive: true, force: true });
this.cloned = false;
logger.info('Cleaned up documentation repository');
} catch (error) {
logger.error('Failed to cleanup docs repository:', error);
}
}
}

337
src/utils/example-generator.ts
View File

@@ -1,267 +1,140 @@
import { logger } from './logger';
interface NodeExample {
nodes: any[];
connections: any;
pinData?: any;
meta?: any;
}
interface NodeParameter {
name: string;
type: string;
default?: any;
options?: any[];
displayOptions?: any;
}
/**
* Generates example workflows and parameters for n8n nodes
*/
export class ExampleGenerator {
/**
* Generate example workflow for a node
* Generate an example workflow from node definition
*/
static generateNodeExample(nodeType: string, nodeData: any): NodeExample {
const nodeName = this.getNodeName(nodeType);
const nodeId = this.generateNodeId();
static generateFromNodeDefinition(nodeDefinition: any): any {
const nodeName = nodeDefinition.displayName || 'Example Node';
const nodeType = nodeDefinition.name || 'n8n-nodes-base.exampleNode';
// Base example structure
const example: NodeExample = {
nodes: [{
parameters: this.generateExampleParameters(nodeType, nodeData),
type: nodeType,
typeVersion: nodeData.typeVersion || 1,
position: [220, 120],
id: nodeId,
name: nodeName
}],
connections: {
[nodeName]: {
main: [[]]
}
},
pinData: {},
meta: {
templateCredsSetupCompleted: true,
instanceId: this.generateInstanceId()
}
return {
name: `${nodeName} Example Workflow`,
nodes: [
{
parameters: this.generateExampleParameters(nodeDefinition),
id: this.generateNodeId(),
name: nodeName,
type: nodeType,
typeVersion: nodeDefinition.version || 1,
position: [250, 300],
},
],
connections: {},
active: false,
settings: {},
tags: ['example', 'generated'],
};
// Add specific configurations based on node type
this.addNodeSpecificConfig(nodeType, example, nodeData);
return example;
}
/**
* Generate example parameters based on node type
* Generate example parameters based on node properties
*/
private static generateExampleParameters(nodeType: string, nodeData: any): any {
static generateExampleParameters(nodeDefinition: any): any {
const params: any = {};
// Extract node name for specific handling
const nodeName = nodeType.split('.').pop()?.toLowerCase() || '';
// Common node examples
switch (nodeName) {
case 'if':
return {
conditions: {
options: {
caseSensitive: true,
leftValue: "",
typeValidation: "strict",
version: 2
},
conditions: [{
id: this.generateNodeId(),
leftValue: "={{ $json }}",
rightValue: "",
operator: {
type: "object",
operation: "notEmpty",
singleValue: true
}
}],
combinator: "and"
},
options: {}
};
case 'webhook':
return {
httpMethod: "POST",
path: "webhook-path",
responseMode: "onReceived",
responseData: "allEntries",
options: {}
};
case 'httprequest':
return {
method: "GET",
url: "https://api.example.com/data",
authentication: "none",
options: {},
headerParametersUi: {
parameter: []
}
};
case 'function':
return {
functionCode: "// Add your JavaScript code here\nreturn $input.all();"
};
case 'set':
return {
mode: "manual",
duplicateItem: false,
values: {
string: [{
name: "myField",
value: "myValue"
}]
}
};
case 'split':
return {
batchSize: 10,
options: {}
};
default:
// Generate generic parameters from node properties
return this.generateGenericParameters(nodeData);
}
}
/**
* Generate generic parameters from node properties
*/
private static generateGenericParameters(nodeData: any): any {
const params: any = {};
if (nodeData.properties) {
for (const prop of nodeData.properties) {
if (prop.default !== undefined) {
params[prop.name] = prop.default;
} else if (prop.type === 'string') {
params[prop.name] = '';
} else if (prop.type === 'number') {
params[prop.name] = 0;
} else if (prop.type === 'boolean') {
params[prop.name] = false;
} else if (prop.type === 'options' && prop.options?.length > 0) {
params[prop.name] = prop.options[0].value;
// If properties are available, generate examples based on them
if (Array.isArray(nodeDefinition.properties)) {
for (const prop of nodeDefinition.properties) {
if (prop.name && prop.type) {
params[prop.name] = this.generateExampleValue(prop);
}
}
}
// Add common parameters based on node type
if (nodeDefinition.displayName?.toLowerCase().includes('trigger')) {
params.pollTimes = {
item: [
{
mode: 'everyMinute',
},
],
};
}
return params;
}
/**
* Add node-specific configurations
* Generate example value based on property definition
*/
private static addNodeSpecificConfig(nodeType: string, example: NodeExample, nodeData: any): void {
const nodeName = nodeType.split('.').pop()?.toLowerCase() || '';
// Add specific connection structures for different node types
switch (nodeName) {
case 'if':
// IF node has true/false outputs
example.connections[example.nodes[0].name] = {
main: [[], []] // Two outputs: true, false
};
break;
private static generateExampleValue(property: any): any {
switch (property.type) {
case 'string':
if (property.name.toLowerCase().includes('url')) {
return 'https://example.com';
}
if (property.name.toLowerCase().includes('email')) {
return 'user@example.com';
}
if (property.name.toLowerCase().includes('name')) {
return 'Example Name';
}
return property.default || 'example-value';
case 'switch':
// Switch node can have multiple outputs
const outputs = nodeData.outputs || 3;
example.connections[example.nodes[0].name] = {
main: Array(outputs).fill([])
};
break;
case 'number':
return property.default || 10;
case 'merge':
// Merge node has multiple inputs
example.nodes[0].position = [400, 120];
// Add dummy input nodes
example.nodes.push({
parameters: {},
type: "n8n-nodes-base.noOp",
typeVersion: 1,
position: [200, 60],
id: this.generateNodeId(),
name: "Input 1"
});
example.nodes.push({
parameters: {},
type: "n8n-nodes-base.noOp",
typeVersion: 1,
position: [200, 180],
id: this.generateNodeId(),
name: "Input 2"
});
example.connections = {
"Input 1": { main: [[{ node: example.nodes[0].name, type: "main", index: 0 }]] },
"Input 2": { main: [[{ node: example.nodes[0].name, type: "main", index: 1 }]] },
[example.nodes[0].name]: { main: [[]] }
};
break;
}
// Add credentials if needed
if (nodeData.credentials?.length > 0) {
example.nodes[0].credentials = {};
for (const cred of nodeData.credentials) {
example.nodes[0].credentials[cred.name] = {
id: this.generateNodeId(),
name: `${cred.name} account`
};
}
case 'boolean':
return property.default !== undefined ? property.default : true;
case 'options':
if (property.options && property.options.length > 0) {
return property.options[0].value;
}
return property.default || '';
case 'collection':
case 'fixedCollection':
return {};
default:
return property.default || null;
}
}
/**
* Extract display name from node type
*/
private static getNodeName(nodeType: string): string {
const parts = nodeType.split('.');
const name = parts[parts.length - 1];
return name.charAt(0).toUpperCase() + name.slice(1);
}
/**
* Generate a random node ID
* Generate a unique node ID
*/
private static generateNodeId(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = Math.random() * 16 | 0;
const v = c === 'x' ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
return Math.random().toString(36).substring(2, 15) +
Math.random().toString(36).substring(2, 15);
}
/**
* Generate instance ID
* Generate example based on node operations
*/
private static generateInstanceId(): string {
return Array(64).fill(0).map(() => Math.floor(Math.random() * 16).toString(16)).join('');
}
/**
* Generate example from node definition
*/
static generateFromNodeDefinition(nodeDefinition: any): NodeExample {
const nodeType = nodeDefinition.description?.name || 'n8n-nodes-base.node';
const nodeData = {
typeVersion: nodeDefinition.description?.version || 1,
properties: nodeDefinition.description?.properties || [],
credentials: nodeDefinition.description?.credentials || [],
outputs: nodeDefinition.description?.outputs || ['main']
};
static generateFromOperations(operations: any[]): any {
const examples: any[] = [];
return this.generateNodeExample(nodeType, nodeData);
if (!operations || operations.length === 0) {
return examples;
}
// Group operations by resource
const resourceMap = new Map<string, any[]>();
for (const op of operations) {
if (!resourceMap.has(op.resource)) {
resourceMap.set(op.resource, []);
}
resourceMap.get(op.resource)!.push(op);
}
// Generate example for each resource
for (const [resource, ops] of resourceMap) {
examples.push({
resource,
operation: ops[0].operation,
description: `Example: ${ops[0].description}`,
parameters: {
resource,
operation: ops[0].operation,
},
});
}
return examples;
}
}

150
src/utils/node-source-extractor.ts
View File

@@ -21,6 +21,8 @@ export class NodeSourceExtractor {
'/n8n-modules',
// Common n8n installation paths
process.env.N8N_CUSTOM_EXTENSIONS || '',
// Additional local path for testing
path.join(process.cwd(), 'node_modules'),
].filter(Boolean);
/**
@@ -75,35 +77,45 @@ export class NodeSourceExtractor {
nodeName: string
): Promise<NodeSourceInfo | null> {
try {
// First, try standard patterns
const standardPatterns = [
`${packageName}/dist/nodes/${nodeName}/${nodeName}.node.js`,
`${packageName}/dist/nodes/${nodeName}.node.js`,
`${packageName}/nodes/${nodeName}/${nodeName}.node.js`,
`${packageName}/nodes/${nodeName}.node.js`,
`${nodeName}/${nodeName}.node.js`,
`${nodeName}.node.js`,
// Try both the provided case and capitalized first letter
const nodeNameVariants = [
nodeName,
nodeName.charAt(0).toUpperCase() + nodeName.slice(1), // Capitalize first letter
nodeName.toLowerCase(), // All lowercase
nodeName.toUpperCase(), // All uppercase
];
// First, try standard patterns with all case variants
for (const nameVariant of nodeNameVariants) {
const standardPatterns = [
`${packageName}/dist/nodes/${nameVariant}/${nameVariant}.node.js`,
`${packageName}/dist/nodes/${nameVariant}.node.js`,
`${packageName}/nodes/${nameVariant}/${nameVariant}.node.js`,
`${packageName}/nodes/${nameVariant}.node.js`,
`${nameVariant}/${nameVariant}.node.js`,
`${nameVariant}.node.js`,
];
// Additional patterns for nested node structures (e.g., agents/Agent)
const nestedPatterns = [
`${packageName}/dist/nodes/*/${nodeName}/${nodeName}.node.js`,
`${packageName}/dist/nodes/**/${nodeName}/${nodeName}.node.js`,
`${packageName}/nodes/*/${nodeName}/${nodeName}.node.js`,
`${packageName}/nodes/**/${nodeName}/${nodeName}.node.js`,
];
// Additional patterns for nested node structures (e.g., agents/Agent)
const nestedPatterns = [
`${packageName}/dist/nodes/*/${nameVariant}/${nameVariant}.node.js`,
`${packageName}/dist/nodes/**/${nameVariant}/${nameVariant}.node.js`,
`${packageName}/nodes/*/${nameVariant}/${nameVariant}.node.js`,
`${packageName}/nodes/**/${nameVariant}/${nameVariant}.node.js`,
];
// Try standard patterns first
for (const pattern of standardPatterns) {
const fullPath = path.join(basePath, pattern);
const result = await this.tryLoadNodeFile(fullPath, packageName, nodeName, basePath);
if (result) return result;
}
// Try standard patterns first
for (const pattern of standardPatterns) {
const fullPath = path.join(basePath, pattern);
const result = await this.tryLoadNodeFile(fullPath, packageName, nodeName, basePath);
if (result) return result;
}
// Try nested patterns (with glob-like search)
for (const pattern of nestedPatterns) {
const result = await this.searchWithGlobPattern(basePath, pattern, packageName, nodeName);
if (result) return result;
// Try nested patterns (with glob-like search)
for (const pattern of nestedPatterns) {
const result = await this.searchWithGlobPattern(basePath, pattern, packageName, nodeName);
if (result) return result;
}
}
// If basePath contains .pnpm, search in pnpm structure
@@ -250,13 +262,49 @@ export class NodeSourceExtractor {
try {
const sourceCode = await fs.readFile(fullPath, 'utf-8');
// Try to find credential file
const credentialPath = fullPath.replace('.node.js', '.credentials.js');
// Try to find credential files
let credentialCode: string | undefined;
// First, try alongside the node file
const credentialPath = fullPath.replace('.node.js', '.credentials.js');
try {
credentialCode = await fs.readFile(credentialPath, 'utf-8');
} catch {
// Credential file is optional
// Try in the credentials directory
const possibleCredentialPaths = [
// Standard n8n structure: dist/credentials/NodeNameApi.credentials.js
path.join(packageBasePath, packageName, 'dist/credentials', `${nodeName}Api.credentials.js`),
path.join(packageBasePath, packageName, 'dist/credentials', `${nodeName}OAuth2Api.credentials.js`),
path.join(packageBasePath, packageName, 'credentials', `${nodeName}Api.credentials.js`),
path.join(packageBasePath, packageName, 'credentials', `${nodeName}OAuth2Api.credentials.js`),
// Without packageName in path
path.join(packageBasePath, 'dist/credentials', `${nodeName}Api.credentials.js`),
path.join(packageBasePath, 'dist/credentials', `${nodeName}OAuth2Api.credentials.js`),
path.join(packageBasePath, 'credentials', `${nodeName}Api.credentials.js`),
path.join(packageBasePath, 'credentials', `${nodeName}OAuth2Api.credentials.js`),
// Try relative to node location
path.join(path.dirname(path.dirname(fullPath)), 'credentials', `${nodeName}Api.credentials.js`),
path.join(path.dirname(path.dirname(fullPath)), 'credentials', `${nodeName}OAuth2Api.credentials.js`),
path.join(path.dirname(path.dirname(path.dirname(fullPath))), 'credentials', `${nodeName}Api.credentials.js`),
path.join(path.dirname(path.dirname(path.dirname(fullPath))), 'credentials', `${nodeName}OAuth2Api.credentials.js`),
];
// Try to find any credential file
const allCredentials: string[] = [];
for (const credPath of possibleCredentialPaths) {
try {
const content = await fs.readFile(credPath, 'utf-8');
allCredentials.push(content);
logger.debug(`Found credential file at: ${credPath}`);
} catch {
// Continue searching
}
}
// If we found credentials, combine them
if (allCredentials.length > 0) {
credentialCode = allCredentials.join('\n\n// --- Next Credential File ---\n\n');
}
}
// Try to get package.json info
@@ -266,12 +314,16 @@ export class NodeSourceExtractor {
path.join(packageBasePath, packageName, 'package.json'),
path.join(path.dirname(path.dirname(fullPath)), 'package.json'),
path.join(path.dirname(path.dirname(path.dirname(fullPath))), 'package.json'),
// Try to go up from the node location to find package.json
path.join(fullPath.split('/dist/')[0], 'package.json'),
path.join(fullPath.split('/nodes/')[0], 'package.json'),
];
for (const packageJsonPath of possiblePackageJsonPaths) {
try {
const packageJson = await fs.readFile(packageJsonPath, 'utf-8');
packageInfo = JSON.parse(packageJson);
logger.debug(`Found package.json at: ${packageJsonPath}`);
break;
} catch {
// Try next path
@@ -295,10 +347,26 @@ export class NodeSourceExtractor {
*/
async listAvailableNodes(category?: string, search?: string): Promise<any[]> {
const nodes: any[] = [];
const seenNodes = new Set<string>(); // Track unique nodes
for (const basePath of this.n8nBasePaths) {
try {
await this.scanDirectoryForNodes(basePath, nodes, category, search);
// Check for n8n-nodes-base specifically
const n8nNodesBasePath = path.join(basePath, 'n8n-nodes-base', 'dist', 'nodes');
try {
await fs.access(n8nNodesBasePath);
await this.scanDirectoryForNodes(n8nNodesBasePath, nodes, category, search, seenNodes);
} catch {
// Try without dist
const altPath = path.join(basePath, 'n8n-nodes-base', 'nodes');
try {
await fs.access(altPath);
await this.scanDirectoryForNodes(altPath, nodes, category, search, seenNodes);
} catch {
// Try the base path directly
await this.scanDirectoryForNodes(basePath, nodes, category, search, seenNodes);
}
}
} catch (error) {
logger.debug(`Failed to scan ${basePath}: ${error}`);
}
@@ -314,7 +382,8 @@ export class NodeSourceExtractor {
dirPath: string,
nodes: any[],
category?: string,
search?: string
search?: string,
seenNodes?: Set<string>
): Promise<void> {
try {
const entries = await fs.readdir(dirPath, { withFileTypes: true });
@@ -330,8 +399,15 @@ export class NodeSourceExtractor {
const descriptionMatch = content.match(/description:\s*['"`]([^'"`]+)['"`]/);
if (nameMatch) {
const nodeName = entry.name.replace('.node.js', '');
// Skip if we've already seen this node
if (seenNodes && seenNodes.has(nodeName)) {
continue;
}
const nodeInfo = {
name: entry.name.replace('.node.js', ''),
name: nodeName,
displayName: nameMatch[1],
description: descriptionMatch ? descriptionMatch[1] : '',
location: fullPath,
@@ -347,6 +423,9 @@ export class NodeSourceExtractor {
}
nodes.push(nodeInfo);
if (seenNodes) {
seenNodes.add(nodeName);
}
}
} catch {
// Skip files we can't read
@@ -354,10 +433,10 @@ export class NodeSourceExtractor {
} else if (entry.isDirectory()) {
// Special handling for .pnpm directories
if (entry.name === '.pnpm') {
await this.scanPnpmDirectory(path.join(dirPath, entry.name), nodes, category, search);
await this.scanPnpmDirectory(path.join(dirPath, entry.name), nodes, category, search, seenNodes);
} else if (entry.name !== 'node_modules') {
// Recursively scan subdirectories
await this.scanDirectoryForNodes(path.join(dirPath, entry.name), nodes, category, search);
await this.scanDirectoryForNodes(path.join(dirPath, entry.name), nodes, category, search, seenNodes);
}
}
}
@@ -373,7 +452,8 @@ export class NodeSourceExtractor {
pnpmPath: string,
nodes: any[],
category?: string,
search?: string
search?: string,
seenNodes?: Set<string>
): Promise<void> {
try {
const entries = await fs.readdir(pnpmPath);
@@ -382,7 +462,7 @@ export class NodeSourceExtractor {
const entryPath = path.join(pnpmPath, entry, 'node_modules');
try {
await fs.access(entryPath);
await this.scanDirectoryForNodes(entryPath, nodes, category, search);
await this.scanDirectoryForNodes(entryPath, nodes, category, search, seenNodes);
} catch {
// Skip if node_modules doesn't exist
}