From d32af279c0ab893eb1669aab32354f14c257d982 Mon Sep 17 00:00:00 2001 From: czlonkowski Date: Sat, 7 Jun 2025 22:11:30 +0000 Subject: [PATCH] Refactor to focused n8n node documentation MCP server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major refactoring to align with actual requirements: - Purpose: Serve n8n node code/documentation to AI agents only - No workflow execution or management features - Complete node information including source code, docs, and examples New features: - Node documentation service with SQLite FTS5 search - Documentation fetcher from n8n-docs repository - Example workflow generator for each node type - Simplified MCP tools focused on node information - Complete database rebuild with all node data MCP Tools: - list_nodes: List available nodes - get_node_info: Get complete node information - search_nodes: Full-text search across nodes - get_node_example: Get usage examples - get_node_source_code: Get source code only - get_node_documentation: Get documentation only - rebuild_database: Rebuild entire database - get_database_statistics: Database stats Database schema includes: - Node source code and metadata - Official documentation from n8n-docs - Generated usage examples - Full-text search capabilities - Category and type filtering Updated README with: - Clear purpose statement - Claude Desktop installation instructions - Complete tool documentation - Troubleshooting guide šŸ¤– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .gitignore | 2 +- README-v2.md | 285 +++++++++++ README.md | 516 ++++++------------- data/nodes-v2-test.db | Bin 0 -> 61440 bytes data/nodes-v2.db | Bin 0 -> 61440 bytes package.json | 6 +- src/db/schema-v2.sql | 99 ++++ src/index-v2.ts | 58 +++ src/mcp/server-v2.ts | 435 ++++++++++++++++ src/mcp/tools-v2.ts | 144 ++++++ src/scripts/rebuild-database-v2.ts | 67 +++ src/services/node-documentation-service.ts | 547 +++++++++++++++++++++ src/utils/documentation-fetcher.ts | 241 +++++++++ src/utils/example-generator.ts | 267 ++++++++++ tests/test-node-documentation-service.js | 88 ++++ tests/test-node-list.js | 26 + tests/test-small-rebuild.js | 62 +++ 17 files changed, 2484 insertions(+), 359 deletions(-) create mode 100644 README-v2.md create mode 100644 data/nodes-v2-test.db create mode 100644 data/nodes-v2.db create mode 100644 src/db/schema-v2.sql create mode 100644 src/index-v2.ts create mode 100644 src/mcp/server-v2.ts create mode 100644 src/mcp/tools-v2.ts create mode 100644 src/scripts/rebuild-database-v2.ts create mode 100644 src/services/node-documentation-service.ts create mode 100644 src/utils/documentation-fetcher.ts create mode 100644 src/utils/example-generator.ts create mode 100644 tests/test-node-documentation-service.js create mode 100644 tests/test-node-list.js create mode 100644 tests/test-small-rebuild.js diff --git a/.gitignore b/.gitignore index c59e97c..c5cba81 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,4 @@ docker-compose.override.yml .lock-wscript .node_repl_history .npmrc -.yarnrc \ No newline at end of file +.yarnrctemp/ diff --git a/README-v2.md b/README-v2.md new file mode 100644 index 0000000..e2ffb75 --- /dev/null +++ b/README-v2.md @@ -0,0 +1,285 @@ +# n8n Node Documentation MCP Server + +An MCP (Model Context Protocol) server that provides n8n node documentation, source code, and usage examples to AI assistants. + +## Purpose + +This MCP server serves as a knowledge base for AI assistants (like Claude) to understand and work with n8n nodes. It provides: + +- **Complete node source code** - The actual implementation of each n8n node +- **Official documentation** - Markdown documentation from the n8n-docs repository +- **Usage examples** - Sample workflow JSON showing how to use each node +- **Search capabilities** - Full-text search across node names, descriptions, and documentation + +## Features + +- šŸ” **Full-text search** - Search nodes by name, description, or documentation content +- šŸ“š **Complete documentation** - Fetches and indexes official n8n documentation +- šŸ’» **Source code access** - Provides full source code for each node +- šŸŽÆ **Usage examples** - Generates example workflows for each node type +- šŸ”„ **Auto-rebuild** - Rebuilds the entire database on startup or on demand +- āš” **Fast SQLite storage** - All data stored locally for instant access + +## Installation + +### Prerequisites + +- Node.js 18+ +- n8n instance (for node extraction) +- Git (for cloning n8n-docs) + +### Setup + +```bash +# Clone the repository +git clone https://github.com/yourusername/n8n-mcp.git +cd n8n-mcp + +# Install dependencies +npm install + +# Build the project +npm run build + +# Rebuild the database with all nodes +npm run db:rebuild +``` + +## Usage with Claude Desktop + +### 1. Configure Claude Desktop + +Add to your Claude Desktop configuration file: + +**macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` +**Windows**: `%APPDATA%\Claude\claude_desktop_config.json` +**Linux**: `~/.config/Claude/claude_desktop_config.json` + +```json +{ + "mcpServers": { + "n8n-nodes": { + "command": "node", + "args": ["/absolute/path/to/n8n-mcp/dist/index-v2.js"], + "env": { + "NODE_DB_PATH": "/absolute/path/to/n8n-mcp/data/nodes.db", + "REBUILD_ON_START": "false" + } + } + } +} +``` + +### 2. Available MCP Tools + +Once configured, you can ask Claude to: + +- **List all n8n nodes**: "Show me all available n8n nodes" +- **Get node information**: "Show me the IF node documentation and code" +- **Search for nodes**: "Find all webhook-related nodes" +- **Get examples**: "Show me an example of using the HTTP Request node" + +### MCP Tools Reference + +#### `list_nodes` +Lists all available n8n nodes with basic information. + +``` +Parameters: +- category (optional): Filter by category +- packageName (optional): Filter by package +- isTrigger (optional): Show only trigger nodes +``` + +#### `get_node_info` +Gets complete information about a specific node including source code, documentation, and examples. + +``` +Parameters: +- nodeType (required): The node type (e.g., "n8n-nodes-base.if", "If", "webhook") +``` + +#### `search_nodes` +Searches for nodes by name, description, or documentation content. + +``` +Parameters: +- query (required): Search query +- category (optional): Filter by category +- hasDocumentation (optional): Only show nodes with docs +- limit (optional): Max results (default: 20) +``` + +#### `get_node_example` +Gets example workflow JSON for a specific node. + +``` +Parameters: +- nodeType (required): The node type +``` + +#### `get_node_source_code` +Gets only the source code of a node. + +``` +Parameters: +- nodeType (required): The node type +- includeCredentials (optional): Include credential definitions +``` + +#### `get_node_documentation` +Gets only the documentation for a node. + +``` +Parameters: +- nodeType (required): The node type +- format (optional): "markdown" or "plain" (default: markdown) +``` + +## Database Management + +### Initial Setup + +```bash +# Build and populate the database +npm run db:rebuild +``` + +### Database Structure + +The SQLite database stores: +- Node source code +- Official documentation from n8n-docs +- Generated usage examples +- Node metadata (category, triggers, webhooks, etc.) + +### Rebuild Process + +The rebuild process: +1. Clears the existing database +2. Fetches latest n8n-docs repository +3. Extracts source code from all n8n nodes +4. Fetches documentation for each node +5. Generates usage examples +6. Stores everything in SQLite with full-text search + +## Example Responses + +### IF Node Example + +When asking for the IF node, the server returns: + +```json +{ + "nodeType": "n8n-nodes-base.if", + "name": "If", + "displayName": "If", + "description": "Route items based on comparison operations", + "sourceCode": "// Full TypeScript source code...", + "documentation": "# If Node\n\nThe If node splits a workflow...", + "exampleWorkflow": { + "nodes": [{ + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [{ + "id": "871274c8-dabf-465a-a6cf-655a1786aa55", + "leftValue": "={{ $json }}", + "rightValue": "", + "operator": { + "type": "object", + "operation": "notEmpty", + "singleValue": true + } + }], + "combinator": "and" + }, + "options": {} + }, + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [220, 120], + "id": "64b5d49f-ac2e-4456-bfa9-2d6eb9c7a624", + "name": "If" + }], + "connections": { + "If": { + "main": [[], []] + } + } + } +} +``` + +## Development + +### Running in Development Mode + +```bash +# Start with auto-reload +npm run dev + +# Run tests +npm test + +# Type checking +npm run typecheck +``` + +### Environment Variables + +```env +# Database location +NODE_DB_PATH=/path/to/nodes.db + +# Rebuild database on server start +REBUILD_ON_START=true + +# Logging +LOG_LEVEL=debug + +# Documentation repository location (optional) +DOCS_REPO_PATH=/path/to/n8n-docs +``` + +## Architecture + +``` +n8n-mcp/ +ā”œā”€ā”€ src/ +ā”‚ ā”œā”€ā”€ mcp/ +ā”‚ ā”‚ ā”œā”€ā”€ server-v2.ts # MCP server implementation +ā”‚ ā”‚ ā””ā”€ā”€ tools-v2.ts # MCP tool definitions +ā”‚ ā”œā”€ā”€ services/ +ā”‚ ā”‚ ā””ā”€ā”€ node-documentation-service.ts # Database service +ā”‚ ā”œā”€ā”€ utils/ +ā”‚ ā”‚ ā”œā”€ā”€ documentation-fetcher.ts # n8n-docs fetcher +ā”‚ ā”‚ ā”œā”€ā”€ example-generator.ts # Example generator +ā”‚ ā”‚ ā””ā”€ā”€ node-source-extractor.ts # Source extractor +ā”‚ ā””ā”€ā”€ scripts/ +ā”‚ ā””ā”€ā”€ rebuild-database-v2.ts # Database rebuild +ā””ā”€ā”€ data/ + ā””ā”€ā”€ nodes.db # SQLite database +``` + +## Troubleshooting + +### Database not found +```bash +npm run db:rebuild +``` + +### No documentation for some nodes +Some nodes may not have documentation in the n8n-docs repository. The server will still provide source code and generated examples. + +### Rebuild takes too long +The initial rebuild processes 500+ nodes and fetches documentation. Subsequent starts use the cached database unless `REBUILD_ON_START=true`. + +## License + +ISC \ No newline at end of file diff --git a/README.md b/README.md index c835aa0..e1d32ac 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,33 @@ -# n8n-MCP Integration +# n8n Node Documentation MCP Server -Complete integration between n8n workflow automation and Model Context Protocol (MCP), enabling bidirectional communication between n8n workflows and AI assistants. +An MCP (Model Context Protocol) server that provides n8n node documentation, source code, and usage examples to AI assistants like Claude. -## Overview +## Purpose -This project provides two main components: -1. **MCP Server**: Allows AI assistants (like Claude) to interact with n8n workflows, execute them, and explore n8n nodes -2. **n8n Custom Node**: Enables n8n workflows to connect to and use MCP servers +This MCP server serves as a comprehensive knowledge base for AI assistants to understand and work with n8n nodes. It provides: -**Important**: The MCP server uses stdio transport and is designed to be invoked by AI assistants like Claude Desktop. It's not a standalone HTTP server but rather a tool that AI assistants can call directly. +- **Complete node source code** - The actual TypeScript/JavaScript implementation of each n8n node +- **Official documentation** - Markdown documentation from the n8n-docs repository +- **Usage examples** - Sample workflow JSON showing how to use each node +- **Search capabilities** - Full-text search across node names, descriptions, and documentation ## Features -- **Bidirectional Integration**: n8n workflows can call MCP tools, and MCP servers can execute n8n workflows -- **Node Source Extraction**: Extract and search source code from any n8n node, including AI Agent nodes -- **SQLite Database**: Full-text search for n8n node documentation and source code (500+ nodes indexed) -- **Production Ready**: Docker-based deployment with persistent storage -- **Comprehensive MCP Tools**: 12+ tools for workflow management, node exploration, and database search -- **Custom n8n Node**: Connect to any MCP server from n8n workflows -- **Auto-indexing**: Automatically builds a searchable database of all n8n nodes on first run +- šŸ” **Full-text search** - Search nodes by name, description, or documentation content +- šŸ“š **Complete documentation** - Fetches and indexes official n8n documentation +- šŸ’» **Source code access** - Provides full source code for each node +- šŸŽÆ **Usage examples** - Auto-generates example workflows for each node type +- šŸ”„ **Database rebuild** - Rebuilds the entire database with latest node information +- āš” **Fast SQLite storage** - All data stored locally for instant access -## Prerequisites +## Installation + +### Prerequisites - Node.js 18+ -- Docker and Docker Compose (for production deployment) -- n8n instance with API access enabled -- (Optional) Claude Desktop for MCP integration +- Git (for cloning n8n-docs) -## Quick Start - -### Installation +### Setup ```bash # Clone the repository @@ -39,415 +37,219 @@ cd n8n-mcp # Install dependencies npm install -# Copy environment template -cp .env.example .env -# Edit .env with your n8n credentials - # Build the project npm run build -# Initialize the database -npm run db:init - -# (Optional) Rebuild database with all nodes -npm run db:rebuild +# Initialize and rebuild the database with all nodes +npm run db:rebuild:v2 ``` -### Development +## Installing in Claude Desktop + +### 1. Build the project first ```bash -# Run tests -npm test - -# Start development server -npm run dev - -# Type checking -npm run typecheck +npm install +npm run build +npm run db:rebuild:v2 # This indexes all n8n nodes ``` -### Production Deployment +### 2. Locate your Claude Desktop configuration -```bash -# Use the automated deployment script -./scripts/deploy-production.sh +- **macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` +- **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` +- **Linux**: `~/.config/Claude/claude_desktop_config.json` -# Or manually with Docker Compose -docker compose -f docker-compose.prod.yml up -d -``` - -See [Production Deployment Guide](docs/PRODUCTION_DEPLOYMENT.md) for detailed instructions. - -## Configuration - -Environment variables (`.env` file): - -```env -# n8n Configuration -N8N_BASIC_AUTH_USER=admin -N8N_BASIC_AUTH_PASSWORD=your-password -N8N_HOST=localhost -N8N_API_KEY=your-api-key - -# Database -NODE_DB_PATH=/app/data/nodes.db - -# Logging -LOG_LEVEL=info -``` - -## Usage - -### Installing the MCP Server in Claude Desktop - -1. **Build the project first:** - ```bash - npm install - npm run build - npm run db:init # Initialize the database - ``` - -2. **Locate your Claude Desktop configuration:** - - **macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` - - **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` - - **Linux**: `~/.config/Claude/claude_desktop_config.json` - -3. **Get your n8n API key:** - - Open n8n in your browser (http://localhost:5678) - - Go to Settings ā†’ API - - Enable "n8n API" if not already enabled - - Generate and copy your API key - -4. **Edit the configuration file:** - ```json - { - "mcpServers": { - "n8n": { - "command": "node", - "args": ["/absolute/path/to/n8n-mcp/dist/index.js"], - "env": { - "N8N_API_URL": "http://localhost:5678", - "N8N_API_KEY": "your-actual-n8n-api-key", - "NODE_DB_PATH": "/absolute/path/to/n8n-mcp/data/nodes.db" - } - } - } - } - ``` - -5. **Important configuration notes:** - - Use **absolute paths** (not relative paths like `~/` or `./`) - - Replace `your-actual-n8n-api-key` with your real n8n API key from step 3 - - Ensure n8n is running at the specified URL (default: http://localhost:5678) - - The `NODE_DB_PATH` should point to your database file - -6. **Restart Claude Desktop** after saving the configuration - -7. **Verify the connection:** - - In Claude, you should see "n8n" in the MCP connections - - Try asking: "Can you list my n8n workflows?" or "Search for webhook nodes in n8n" - -#### Example Configuration - -Here's a complete example for macOS with n8n-mcp installed in your home directory: +### 3. Edit the configuration file ```json { "mcpServers": { - "n8n": { + "n8n-nodes": { "command": "node", - "args": ["/Users/yourusername/n8n-mcp/dist/index.js"], + "args": ["/absolute/path/to/n8n-mcp/dist/index-v2.js"], "env": { - "N8N_API_URL": "http://localhost:5678", - "N8N_API_KEY": "n8n_api_key_from_settings", - "NODE_DB_PATH": "/Users/yourusername/n8n-mcp/data/nodes.db" + "NODE_DB_PATH": "/absolute/path/to/n8n-mcp/data/nodes-v2.db" } } } } ``` -### Available MCP Tools +**Important**: Use absolute paths, not relative paths like `~/` or `./` -#### Workflow Management -- `execute_workflow` - Execute an n8n workflow by ID -- `list_workflows` - List all workflows with filtering options -- `get_workflow` - Get detailed workflow information -- `create_workflow` - Create new workflows programmatically -- `update_workflow` - Update existing workflows -- `delete_workflow` - Delete workflows -- `get_executions` - Get workflow execution history -- `get_execution_data` - Get detailed execution data +### 4. Restart Claude Desktop -#### Node Exploration & Search -- `list_available_nodes` - List all available n8n nodes -- `get_node_source_code` - Extract source code of any n8n node -- `search_nodes` - Search nodes by name or content using full-text search -- `extract_all_nodes` - Extract and index all nodes to database -- `get_node_statistics` - Get database statistics (total nodes, packages, etc.) +After saving the configuration, completely quit and restart Claude Desktop. -### Using the n8n MCP Node +### 5. Verify the connection -1. Install the node in n8n: - ```bash - # Copy to n8n custom nodes directory - cp -r dist/n8n/* ~/.n8n/custom/ - # Or use the install script - ./scripts/install-n8n-node.sh - ``` -2. Restart n8n -3. The MCP node will appear in the nodes panel -4. Configure with your MCP server credentials -5. Select operations: Call Tool, List Tools, Read Resource, etc. +In Claude, you should see "n8n-nodes" in the MCP connections. Try asking: +- "What n8n nodes are available?" +- "Show me how to use the IF node in n8n" +- "Search for webhook nodes in n8n" +- "Show me the source code for the HTTP Request node" -### Example n8n Workflow +## Available MCP Tools + +### `list_nodes` +Lists all available n8n nodes with basic information. +- Parameters: `category`, `packageName`, `isTrigger` (all optional) + +### `get_node_info` +Gets complete information about a specific node including source code, documentation, and examples. +- Parameters: `nodeType` (required) - e.g., "n8n-nodes-base.if", "If", "webhook" + +### `search_nodes` +Searches for nodes by name, description, or documentation content. +- Parameters: `query` (required), `category`, `hasDocumentation`, `limit` (optional) + +### `get_node_example` +Gets example workflow JSON for a specific node. +- Parameters: `nodeType` (required) + +### `get_node_source_code` +Gets only the source code of a node. +- Parameters: `nodeType` (required), `includeCredentials` (optional) + +### `get_node_documentation` +Gets only the documentation for a node. +- Parameters: `nodeType` (required), `format` (optional: "markdown" or "plain") + +### `rebuild_database` +Rebuilds the entire node database with latest information. +- Parameters: `includeDocumentation` (optional, default: true) + +### `get_database_statistics` +Gets statistics about the node database. +- No parameters required + +## Example Usage + +When you ask Claude about the IF node, it will use the MCP tools to provide: ```json { - "name": "MCP AI Assistant", - "nodes": [ - { - "name": "MCP", - "type": "mcp", + "nodeType": "n8n-nodes-base.if", + "name": "If", + "displayName": "If", + "description": "Route items based on comparison operations", + "sourceCode": "// Full TypeScript source code...", + "documentation": "# If Node\n\nThe If node splits a workflow...", + "exampleWorkflow": { + "nodes": [{ "parameters": { - "operation": "callTool", - "toolName": "generate_text", - "toolArguments": "{\"prompt\": \"Write a summary\"}" + "conditions": { + "conditions": [{ + "leftValue": "={{ $json }}", + "rightValue": "", + "operator": { + "type": "object", + "operation": "notEmpty" + } + }] + } + }, + "type": "n8n-nodes-base.if", + "position": [220, 120], + "name": "If" + }], + "connections": { + "If": { + "main": [[], []] } } - ] + } } ``` -## Architecture +## Database Management -### Components - -1. **MCP Server** (`src/mcp/`): Exposes n8n operations as MCP tools -2. **n8n Custom Node** (`src/n8n/`): Allows n8n to connect to MCP servers -3. **SQLite Storage** (`src/services/`): Persistent storage with full-text search -4. **Bridge Layer** (`src/utils/`): Converts between n8n and MCP formats - -### Database Management - -The SQLite database stores n8n node documentation and source code with full-text search capabilities: +### Initial Setup +The database is automatically created when you run: ```bash -# Initialize empty database -npm run db:init - -# Rebuild the entire database (indexes all nodes) -npm run db:rebuild - -# In production -./scripts/manage-production.sh rebuild-db - -# View statistics -./scripts/manage-production.sh db-stats +npm run db:rebuild:v2 ``` -#### Search Examples +This process: +1. Clones/updates the n8n-docs repository +2. Extracts source code from all n8n nodes +3. Fetches documentation for each node +4. Generates usage examples +5. Stores everything in SQLite with full-text search -```javascript -// Search for nodes by name -await mcp.callTool('search_nodes', { query: 'webhook' }) +### Manual Rebuild -// Search in specific package -await mcp.callTool('search_nodes', { - query: 'http', - packageName: 'n8n-nodes-base' -}) - -// Get database statistics -await mcp.callTool('get_node_statistics', {}) +To update the database with latest nodes: +```bash +npm run db:rebuild:v2 ``` -## Testing +### Database Location -### Run All Tests -```bash -npm test -``` +The SQLite database is stored at: `data/nodes-v2.db` -### Test Specific Features -```bash -# Test node extraction -node tests/test-mcp-extraction.js - -# Test database search -node tests/test-sqlite-search.js - -# Test AI Agent extraction -./scripts/test-ai-agent-extraction.sh -``` - -## API Reference - -### MCP Tools - -#### execute_workflow -Execute an n8n workflow by ID. - -Parameters: -- `workflowId` (string, required): The workflow ID -- `data` (object, optional): Input data for the workflow - -#### list_workflows -List all available workflows. - -Parameters: -- `active` (boolean, optional): Filter by active status -- `tags` (array, optional): Filter by tags - -#### get_node_source_code -Extract source code of any n8n node. - -Parameters: -- `nodeType` (string, required): The node type identifier (e.g., `n8n-nodes-base.Webhook`) -- `includeCredentials` (boolean, optional): Include credential type definitions if available - -#### search_nodes -Search nodes using full-text search in the SQLite database. - -Parameters: -- `query` (string, optional): Search term for node names/descriptions -- `packageName` (string, optional): Filter by package name -- `nodeType` (string, optional): Filter by node type pattern -- `hasCredentials` (boolean, optional): Filter nodes with credentials -- `limit` (number, optional): Limit results (default: 20) - -#### extract_all_nodes -Extract and index all available nodes to the database. - -Parameters: -- `packageFilter` (string, optional): Filter by package name -- `limit` (number, optional): Limit number of nodes to extract - -#### get_node_statistics -Get database statistics including total nodes, packages, and size. - -No parameters required. - -#### list_available_nodes -List all available n8n nodes in the system. - -Parameters: -- `category` (string, optional): Filter by category -- `search` (string, optional): Search term to filter nodes - -### MCP Resources - -- `workflow://active` - List of active workflows -- `workflow://all` - List of all workflows -- `execution://recent` - Recent execution history -- `credentials://types` - Available credential types -- `nodes://available` - Available n8n nodes -- `nodes://source/{nodeType}` - Source code of specific n8n node - -### MCP Prompts - -- `create_workflow_prompt` - Generate workflow creation prompts -- `debug_workflow_prompt` - Debug workflow issues -- `optimize_workflow_prompt` - Optimize workflow performance -- `explain_workflow_prompt` - Explain workflow functionality - -## Management - -Use the management script for production operations: +## Development ```bash -# Check status -./scripts/manage-production.sh status +# Run in development mode +npm run dev:v2 -# View logs -./scripts/manage-production.sh logs +# Run tests +npm run test:v2 -# Create backup -./scripts/manage-production.sh backup - -# Update services -./scripts/manage-production.sh update +# Type checking +npm run typecheck ``` ## Troubleshooting -### Common Issues +### Claude Desktop doesn't show the MCP server -1. **Claude Desktop doesn't show the MCP server** - - Ensure you've restarted Claude Desktop after editing the config - - Check the config file is valid JSON (no trailing commas) - - Verify the absolute paths are correct - - Check Claude's developer console for errors (Help ā†’ Developer) +1. Ensure you've restarted Claude Desktop after editing the config +2. Check the config file is valid JSON (no trailing commas) +3. Verify the absolute paths are correct +4. Check Claude's developer console for errors (Help ā†’ Developer) -2. **"Connection failed" in Claude** - - Verify n8n is running at the specified URL - - Check the API key is correct (get it from n8n settings ā†’ API) - - Ensure the MCP server is built (`npm run build`) - - Try running the server manually: `N8N_API_KEY=your-key node dist/index.js` +### "Connection failed" in Claude -3. **MCP server keeps restarting in Docker** - - This is expected behavior. The MCP server uses stdio transport and waits for input from AI assistants. - - For testing, use the development mode or invoke through Claude Desktop. +1. Ensure the MCP server is built (`npm run build`) +2. Check that the database exists (`data/nodes-v2.db`) +3. Verify the NODE_DB_PATH in Claude config points to the correct database file -4. **Database not found** - - Run `npm run db:init` to create the database - - Run `npm run db:rebuild` to populate with all nodes - - Ensure NODE_DB_PATH in Claude config points to the actual database file +### Database rebuild fails -5. **n8n API connection failed** - - Verify n8n is running and accessible - - Check API key in `.env` file - - Ensure n8n API is enabled in settings - - Try accessing `http://localhost:5678/api/v1/workflows` with your API key +Some nodes may fail to extract (deprecated nodes, triggers without main node file). This is normal. The rebuild will continue and index all available nodes. -6. **Node extraction fails** - - Ensure Docker volume mounts are correct - - Check read permissions on node_modules directory - - Run `npm run db:rebuild` to re-index all nodes +### No documentation for some nodes -## Documentation - -- [Production Deployment Guide](docs/PRODUCTION_DEPLOYMENT.md) -- [AI Agent Extraction Test](docs/AI_AGENT_EXTRACTION_TEST.md) - -## Security - -- Token-based authentication for n8n API -- Read-only access to node source files -- Isolated Docker containers -- Persistent volume encryption (optional) +Not all nodes have documentation in the n8n-docs repository. The server will still provide source code and generated examples for these nodes. ## Project Structure ``` n8n-mcp/ ā”œā”€ā”€ src/ -ā”‚ ā”œā”€ā”€ mcp/ # MCP server implementation -ā”‚ ā”œā”€ā”€ n8n/ # n8n custom node -ā”‚ ā”œā”€ā”€ services/ # SQLite storage service -ā”‚ ā”œā”€ā”€ utils/ # Utilities and helpers -ā”‚ ā””ā”€ā”€ scripts/ # Database management scripts -ā”œā”€ā”€ tests/ # Test suite -ā”œā”€ā”€ docs/ # Documentation -ā”œā”€ā”€ scripts/ # Deployment and management scripts -ā””ā”€ā”€ data/ # SQLite database (created on init) +ā”‚ ā”œā”€ā”€ mcp/ +ā”‚ ā”‚ ā”œā”€ā”€ server-v2.ts # MCP server implementation +ā”‚ ā”‚ ā””ā”€ā”€ tools-v2.ts # MCP tool definitions +ā”‚ ā”œā”€ā”€ services/ +ā”‚ ā”‚ ā””ā”€ā”€ node-documentation-service.ts # Database service +ā”‚ ā”œā”€ā”€ utils/ +ā”‚ ā”‚ ā”œā”€ā”€ documentation-fetcher.ts # n8n-docs fetcher +ā”‚ ā”‚ ā”œā”€ā”€ example-generator.ts # Example generator +ā”‚ ā”‚ ā””ā”€ā”€ node-source-extractor.ts # Source extractor +ā”‚ ā””ā”€ā”€ scripts/ +ā”‚ ā””ā”€ā”€ rebuild-database-v2.ts # Database rebuild +ā””ā”€ā”€ data/ + ā””ā”€ā”€ nodes-v2.db # SQLite database ``` -## Contributing - -Contributions are welcome! Please: -1. Fork the repository -2. Create a feature branch -3. Add tests for new features -4. Ensure all tests pass -5. Submit a pull request - ## License -ISC License +ISC ## Support -- GitHub Issues: [Report bugs or request features](https://github.com/yourusername/n8n-mcp/issues) -- n8n Community: [n8n.io/community](https://community.n8n.io/) -- MCP Documentation: [modelcontextprotocol.io](https://modelcontextprotocol.io/) +For issues and questions, please use the GitHub issue tracker. \ No newline at end of file diff --git a/data/nodes-v2-test.db b/data/nodes-v2-test.db new file mode 100644 index 0000000000000000000000000000000000000000..e10dfe0f95c0ac58835b7a5ac2f9e26bc35aebfd GIT binary patch literal 61440 zcmeI*+i%;}9S3k8-6daW7sEKLEy{-hwpKflFS4Bm0p`?9DomCVS!xn&m`00~%~}!_ zl8T*f*d`JkfJV;+7v~R#r-l?B%X74E+6t7 z9+HZ4biZhMrrNZfw(hB0(vrlO^fOhJBx!+OGQ9@N5`BmbR_L9Le4h1jLAv(QPZ#9> zNUueHD9!&`{`>sm++XI7GXKr|HFH1xo3xzzbL!3H-x9xy|0DLx=mjOh@_!(3S${35 ztgo|Yk3IdkWttbBqc=Ru?$ljR_uR>K`Q3`PU)9v={_UcsPR6RM8CA6mwNR>RceRRo zuTnVNuRKzJu02xsYt?e0M4KOKrRo}`^K4IV)jPIfy5R<;a#bzWibWnTV(RCXcUm{> z#@D7XFZ6b|H6t&-tOk`;cdZZ2spa`;cu}*{j5OHLsWTmrMYV9KsR!Df{aUfAzFVtQ zsC{)FcvRg#yqC%L_2s0pyUQ+hAxs7I+GdAv1+v|98m2onaB<9fYLagR?uxS1s`ikO zN1z*ayKQ;(Q{6okksQBidW}=lsMARCB5l0M%bsJMoS2Tk^Yos0AGD)V zxXX`??qD{2&er&2x)HfegU`rtjSqN$F3PKVho z)ovNrEaN7nE|iY6N|o=}@-X$PNVg^e>fY0?Ngwq#{cTuox1~R=2WzI=aI7xhA!}-b z_Uno5Jf%cqd#bHFXNG-FDfE`5yQZ6^Qs3V%*0dvawPT)BJt%j2=gk`w##cpwJYu|1 z5i(jZ4;U6MqGwcq#|8E=CZktY&qfF1HT6kW zeNd>rSFTmnO8G(IpgW&bu3u+;d7?Dx1AM?7<`)Fa&nhp7Us;`g8dy`^57pbn^6l*J z=90?F3hUpTDA6)5hNaQU+^mu)Ze`UqPk6ucOsBo3c8O(ew4I~T*@+f^kVz`p?35PM zFg>gQjq-YCRthgTNv0*tu3GWvw)!KeBIJ<_iYVD{{ zx~tOO&+`*35BCQczQTtWdLJ8H`1;7uLaX7x!lR}SD-;t9Dm*_r%se_v-4c};38;%9 z^_FF1vcE_rl)S>8S{=i@puORr%j=d~9~}bHLE8$Y1MMMA4xUdI4*jdcO|yk$Ldhyj zpU9jyS5zrABU_$em?2-wK1?K(ErsOprW$4%24!pUgtDQq$CEh+ zrx1~GVphgMqLF-qknC|RsoYRlpH1Z3)f;E}i7C!nBJcQ&yhjrb^B)b(-is!bx8G)u z!~y9(ZqcPd-8CQg%nnUQ##duODgK;qB63w6fV5@yqYqe8Q55#4t>Hl~Y8e^ZUIcxk zc?;h{9Pj?rrB*83uW4g57_rlzo7aqXiEv^w(l5fWLFWhq+}`nXVHWj>FadRt^>y{g zpHT7Rfdc&L6^+k^DbkJAu;G{nbtj8vykP>rS5e!-nb-IxyxA?YbK;#&ov-?JI2i5B z6g0hahZP@>?m4Y6#?MakXVGfUr2~zEf?0L_+;+~IE&Du7K8Wb*4)rk_f88*u>)2h> z@hp=j(lm)|>tT#v#-ZuyVq8_#(L++C8WVUzxze00+?>l@)cK}iP%vVJ>wXD$TKb7g z9|^WTlpkI~3})*&z6*wfPRKkNe{LS1+Vc)!T1)i&vhsomw}2A2shI)CQ1=yj=G zFSr*I5@A((A|c)aP2E$^TUB z9zFe|duDA{<-vwZw<+j38r==CT>8jwfQSctZrMAfdm(gdAh)U32b7Sv#XK$yumEUL>`K^uZ+;*<9wcXe-Hgh+cH}d(szOyu5 zsmv{a}Fr1Rwwb2tWV=5P$##AOHafK;YFE z_(Ak~di)jv-M8?c|9>XQpS}7GL<=AQ0SG_<0uX=z1Rwwb2tWV=5EvCmv8a+3-(T>b z|9>jUpN_`kGXx+20SG_<0uX=z1Rwwb2tWV=udcvyG&}JX0RQ^`b4mXE)om4O4gm;2 z00Izz00bZa0SG_<0uX?}RDl#d{};c6z`y?gSdu@Unht>wfB*y_009U<00Izz00bZa z0SLTYfrD6oA^mNCX`p}o|Ai!f@p2o3(jfo=2tWV=5P$##AOHafKmY;|_$~zE(dBfZ z>0keUDal`c7iAz%2tWV=5P$##AOHafKmY;|fB*zuwm>wRO7q|UAC-%e{7?B`^o9ij z5P$##AOHafKmY;|fB*y_0D-R&=r2C|11&5jU8VFM9L&> zbZR=ZQ7ppZj@|Q2ooF?y^FR%00Izz00bZa0SG_<0uX=z I1bl)20R+S^I{*Lx literal 0 HcmV?d00001 diff --git a/data/nodes-v2.db b/data/nodes-v2.db new file mode 100644 index 0000000000000000000000000000000000000000..7e42a15d0af28fe65bf99f70e6678b556b460114 GIT binary patch literal 61440 zcmeI)%}?8A90zcl&?JPHuuht#RaKuGc!iFFx@xVE7K)+KLehq$wAwV!BnDiPIM_}D z-82c*^)~JQ*kSui_8089;|{y*uxXe5p2v>kgalgcGT7IGB+rYVm(Szp@nc1>`8a2J zCfl){y5X@UXeuKZ@CPepPhRSZ1w6!SW+s z11seHgw%?*p3PGeR<)c)@oUAxMsUiG=d!JbT2W(;-KNWInc21KEvtH;a%c0KTCv0j zw$RNzEAq{W2;Dnsn)K1A(`(gon>FJ|4??C}ajYgEkU3VNaox3@Bg)h>Qgy@Gui9fzeUizQwM{nLFx%7)DxKbW_d1pFps0{1^j0cjq80OqZsj6*NCS9UrxEv4 zT7LCdN(=tutzQ1>8T{3NKVbe>{M3Y#{FG?IE4Q}^))`H(f+hrGcd7~h>oKiOigo#T zT&x=YoS_MzSM0`)wc8(_?94H-F3}qo&p?vtE ze4Os9jphJfFuUai5fj7e3({w12j2$fnER5g&vbhdEmpY5KiW)pEmRb->DaNb?fq~wrdd4_I*JfuF2$CV{T zZuJ!+K%)FM}u1Y zJzrnqthfN_O!f71IiVyx$sIPlH@W51!W6$a&EG|{mP;2J zMFqQRy=^=DJ2kr3Eh&8)@1^){QX5uVcgX46&{<7QG}z zt{#CGR4VPs!qd6jq0UbYqk2Mh6S zG1X%9keTEUnv?v6KiT7e00bZa0SG_<0uX=z1Rwwb2teS{3Z!_?@ce&i#}^%g00bZa z0SG_<0uX=z1Rwwb2yg-Z{lA?4O`;zh5P$##AOHafKmY;|fB*y_009VGe1Ui6*!)=J zZI { + logger.info('Received SIGINT, shutting down MCP server...'); + process.exit(0); +}); + +process.on('SIGTERM', () => { + logger.info('Received SIGTERM, shutting down MCP server...'); + process.exit(0); +}); + +main().catch((error) => { + logger.error('Unhandled error:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/src/mcp/server-v2.ts b/src/mcp/server-v2.ts new file mode 100644 index 0000000..dbe1b79 --- /dev/null +++ b/src/mcp/server-v2.ts @@ -0,0 +1,435 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { + CallToolRequestSchema, + ErrorCode, + ListResourcesRequestSchema, + ListToolsRequestSchema, + McpError, + ReadResourceRequestSchema, +} from '@modelcontextprotocol/sdk/types.js'; +import { NodeDocumentationService } from '../services/node-documentation-service'; +import { nodeDocumentationTools } from './tools-v2'; +import { logger } from '../utils/logger'; +import { MCPServerConfig } from '../types'; + +/** + * MCP Server focused on serving n8n node documentation and code + */ +export class N8NDocumentationMCPServer { + private server: Server; + private nodeService: NodeDocumentationService; + + constructor(config: MCPServerConfig) { + logger.info('Initializing n8n Documentation MCP server', { config }); + + this.server = new Server( + { + name: 'n8n-node-documentation', + version: '2.0.0', + }, + { + capabilities: { + tools: {}, + resources: {}, + }, + } + ); + + this.nodeService = new NodeDocumentationService(); + this.setupHandlers(); + } + + private setupHandlers(): void { + // List available tools + this.server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: nodeDocumentationTools, + })); + + // List available resources + this.server.setRequestHandler(ListResourcesRequestSchema, async () => ({ + resources: [ + { + uri: 'nodes://list', + name: 'Available n8n Nodes', + description: 'List of all available n8n nodes', + mimeType: 'application/json', + }, + { + uri: 'nodes://statistics', + name: 'Database Statistics', + description: 'Statistics about the node documentation database', + mimeType: 'application/json', + }, + ], + })); + + // Read resources + this.server.setRequestHandler(ReadResourceRequestSchema, async (request) => { + const { uri } = request.params; + + try { + if (uri === 'nodes://list') { + const nodes = await this.nodeService.listNodes(); + return { + contents: [ + { + uri, + mimeType: 'application/json', + text: JSON.stringify(nodes.map(n => ({ + nodeType: n.nodeType, + name: n.name, + displayName: n.displayName, + category: n.category, + description: n.description, + hasDocumentation: !!n.documentation, + hasExample: !!n.exampleWorkflow, + })), null, 2), + }, + ], + }; + } + + if (uri === 'nodes://statistics') { + const stats = this.nodeService.getStatistics(); + return { + contents: [ + { + uri, + mimeType: 'application/json', + text: JSON.stringify(stats, null, 2), + }, + ], + }; + } + + // Handle specific node URIs like nodes://info/n8n-nodes-base.if + const nodeMatch = uri.match(/^nodes:\/\/info\/(.+)$/); + if (nodeMatch) { + const nodeType = nodeMatch[1]; + const nodeInfo = await this.nodeService.getNodeInfo(nodeType); + + if (!nodeInfo) { + throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${nodeType}`); + } + + return { + contents: [ + { + uri, + mimeType: 'application/json', + text: JSON.stringify(nodeInfo, null, 2), + }, + ], + }; + } + + throw new McpError(ErrorCode.InvalidRequest, `Unknown resource: ${uri}`); + } catch (error) { + logger.error('Resource read error:', error); + throw error instanceof McpError ? error : new McpError( + ErrorCode.InternalError, + `Failed to read resource: ${error instanceof Error ? error.message : String(error)}` + ); + } + }); + + // Handle tool calls + this.server.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name, arguments: args } = request.params; + + try { + switch (name) { + case 'list_nodes': + return await this.handleListNodes(args); + + case 'get_node_info': + return await this.handleGetNodeInfo(args); + + case 'search_nodes': + return await this.handleSearchNodes(args); + + case 'get_node_example': + return await this.handleGetNodeExample(args); + + case 'get_node_source_code': + return await this.handleGetNodeSourceCode(args); + + case 'get_node_documentation': + return await this.handleGetNodeDocumentation(args); + + case 'rebuild_database': + return await this.handleRebuildDatabase(args); + + case 'get_database_statistics': + return await this.handleGetStatistics(); + + default: + throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${name}`); + } + } catch (error) { + logger.error(`Tool execution error (${name}):`, error); + throw error instanceof McpError ? error : new McpError( + ErrorCode.InternalError, + `Tool execution failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + }); + } + + private async handleListNodes(args: any): Promise { + const nodes = await this.nodeService.listNodes(); + + // Apply filters + let filtered = nodes; + + if (args.category) { + filtered = filtered.filter(n => n.category === args.category); + } + + if (args.packageName) { + filtered = filtered.filter(n => n.packageName === args.packageName); + } + + if (args.isTrigger !== undefined) { + filtered = filtered.filter(n => n.isTrigger === args.isTrigger); + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify(filtered.map(n => ({ + nodeType: n.nodeType, + name: n.name, + displayName: n.displayName, + category: n.category, + description: n.description, + packageName: n.packageName, + hasDocumentation: !!n.documentation, + hasExample: !!n.exampleWorkflow, + isTrigger: n.isTrigger, + isWebhook: n.isWebhook, + })), null, 2), + }, + ], + }; + } + + private async handleGetNodeInfo(args: any): Promise { + if (!args.nodeType) { + throw new McpError(ErrorCode.InvalidParams, 'nodeType is required'); + } + + const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType); + + if (!nodeInfo) { + throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`); + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + nodeType: nodeInfo.nodeType, + name: nodeInfo.name, + displayName: nodeInfo.displayName, + description: nodeInfo.description, + category: nodeInfo.category, + packageName: nodeInfo.packageName, + sourceCode: nodeInfo.sourceCode, + credentialCode: nodeInfo.credentialCode, + documentation: nodeInfo.documentation, + documentationUrl: nodeInfo.documentationUrl, + exampleWorkflow: nodeInfo.exampleWorkflow, + exampleParameters: nodeInfo.exampleParameters, + propertiesSchema: nodeInfo.propertiesSchema, + isTrigger: nodeInfo.isTrigger, + isWebhook: nodeInfo.isWebhook, + }, null, 2), + }, + ], + }; + } + + private async handleSearchNodes(args: any): Promise { + if (!args.query) { + throw new McpError(ErrorCode.InvalidParams, 'query is required'); + } + + const results = await this.nodeService.searchNodes({ + query: args.query, + category: args.category, + limit: args.limit || 20, + }); + + // Filter by documentation if requested + let filtered = results; + if (args.hasDocumentation) { + filtered = filtered.filter(n => !!n.documentation); + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify(filtered.map(n => ({ + nodeType: n.nodeType, + name: n.name, + displayName: n.displayName, + category: n.category, + description: n.description, + hasDocumentation: !!n.documentation, + hasExample: !!n.exampleWorkflow, + })), null, 2), + }, + ], + }; + } + + private async handleGetNodeExample(args: any): Promise { + if (!args.nodeType) { + throw new McpError(ErrorCode.InvalidParams, 'nodeType is required'); + } + + const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType); + + if (!nodeInfo) { + throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`); + } + + if (!nodeInfo.exampleWorkflow) { + return { + content: [ + { + type: 'text', + text: `No example available for node: ${args.nodeType}`, + }, + ], + }; + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify(nodeInfo.exampleWorkflow, null, 2), + }, + ], + }; + } + + private async handleGetNodeSourceCode(args: any): Promise { + if (!args.nodeType) { + throw new McpError(ErrorCode.InvalidParams, 'nodeType is required'); + } + + const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType); + + if (!nodeInfo) { + throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`); + } + + const response: any = { + nodeType: nodeInfo.nodeType, + sourceCode: nodeInfo.sourceCode, + }; + + if (args.includeCredentials && nodeInfo.credentialCode) { + response.credentialCode = nodeInfo.credentialCode; + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify(response, null, 2), + }, + ], + }; + } + + private async handleGetNodeDocumentation(args: any): Promise { + if (!args.nodeType) { + throw new McpError(ErrorCode.InvalidParams, 'nodeType is required'); + } + + const nodeInfo = await this.nodeService.getNodeInfo(args.nodeType); + + if (!nodeInfo) { + throw new McpError(ErrorCode.InvalidRequest, `Node not found: ${args.nodeType}`); + } + + if (!nodeInfo.documentation) { + return { + content: [ + { + type: 'text', + text: `No documentation available for node: ${args.nodeType}`, + }, + ], + }; + } + + const content = args.format === 'plain' + ? nodeInfo.documentation.replace(/[#*`]/g, '') + : nodeInfo.documentation; + + return { + content: [ + { + type: 'text', + text: content, + }, + ], + }; + } + + private async handleRebuildDatabase(args: any): Promise { + logger.info('Starting database rebuild...'); + + const stats = await this.nodeService.rebuildDatabase(); + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + message: 'Database rebuild complete', + stats, + }, null, 2), + }, + ], + }; + } + + private async handleGetStatistics(): Promise { + const stats = this.nodeService.getStatistics(); + + return { + content: [ + { + type: 'text', + text: JSON.stringify(stats, null, 2), + }, + ], + }; + } + + async start(): Promise { + logger.info('Starting n8n Documentation MCP server...'); + + const transport = new StdioServerTransport(); + await this.server.connect(transport); + + logger.info('n8n Documentation MCP server started successfully'); + } + + async stop(): Promise { + logger.info('Stopping n8n Documentation MCP server...'); + await this.server.close(); + this.nodeService.close(); + logger.info('Server stopped'); + } +} \ No newline at end of file diff --git a/src/mcp/tools-v2.ts b/src/mcp/tools-v2.ts new file mode 100644 index 0000000..14ed402 --- /dev/null +++ b/src/mcp/tools-v2.ts @@ -0,0 +1,144 @@ +import { ToolDefinition } from '../types'; + +/** + * Simplified MCP tools focused on serving n8n node documentation and code + */ +export const nodeDocumentationTools: ToolDefinition[] = [ + { + name: 'list_nodes', + description: 'List all available n8n nodes with basic information', + inputSchema: { + type: 'object', + properties: { + category: { + type: 'string', + description: 'Filter by category (e.g., "Core Nodes", "Flow", "Data Transformation")', + }, + packageName: { + type: 'string', + description: 'Filter by package name (e.g., "n8n-nodes-base")', + }, + isTrigger: { + type: 'boolean', + description: 'Filter to show only trigger nodes', + }, + }, + }, + }, + { + name: 'get_node_info', + description: 'Get complete information about a specific n8n node including source code, documentation, and examples', + inputSchema: { + type: 'object', + properties: { + nodeType: { + type: 'string', + description: 'The node type or name (e.g., "n8n-nodes-base.if", "If", "webhook")', + }, + }, + required: ['nodeType'], + }, + }, + { + name: 'search_nodes', + description: 'Search for n8n nodes by name, description, or documentation content', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Search query (searches in node names, descriptions, and documentation)', + }, + category: { + type: 'string', + description: 'Filter by category', + }, + hasDocumentation: { + type: 'boolean', + description: 'Filter to show only nodes with documentation', + }, + limit: { + type: 'number', + description: 'Maximum number of results to return', + default: 20, + }, + }, + required: ['query'], + }, + }, + { + name: 'get_node_example', + description: 'Get example workflow/usage for a specific n8n node', + inputSchema: { + type: 'object', + properties: { + nodeType: { + type: 'string', + description: 'The node type or name', + }, + }, + required: ['nodeType'], + }, + }, + { + name: 'get_node_source_code', + description: 'Get only the source code of a specific n8n node', + inputSchema: { + type: 'object', + properties: { + nodeType: { + type: 'string', + description: 'The node type or name', + }, + includeCredentials: { + type: 'boolean', + description: 'Include credential type definitions if available', + default: false, + }, + }, + required: ['nodeType'], + }, + }, + { + name: 'get_node_documentation', + description: 'Get only the documentation for a specific n8n node', + inputSchema: { + type: 'object', + properties: { + nodeType: { + type: 'string', + description: 'The node type or name', + }, + format: { + type: 'string', + enum: ['markdown', 'plain'], + description: 'Documentation format', + default: 'markdown', + }, + }, + required: ['nodeType'], + }, + }, + { + name: 'rebuild_database', + description: 'Rebuild the entire node database with latest information from n8n and documentation', + inputSchema: { + type: 'object', + properties: { + includeDocumentation: { + type: 'boolean', + description: 'Include documentation from n8n-docs repository', + default: true, + }, + }, + }, + }, + { + name: 'get_database_statistics', + description: 'Get statistics about the node database', + inputSchema: { + type: 'object', + properties: {}, + }, + }, +]; \ No newline at end of file diff --git a/src/scripts/rebuild-database-v2.ts b/src/scripts/rebuild-database-v2.ts new file mode 100644 index 0000000..d354421 --- /dev/null +++ b/src/scripts/rebuild-database-v2.ts @@ -0,0 +1,67 @@ +#!/usr/bin/env node + +import { NodeDocumentationService } from '../services/node-documentation-service'; +import { logger } from '../utils/logger'; + +async function rebuildDatabase() { + console.log('šŸ”„ Starting complete database rebuild...\n'); + + const service = new NodeDocumentationService(); + + try { + const startTime = Date.now(); + + console.log('1ļøāƒ£ Initializing services...'); + console.log('2ļøāƒ£ Fetching n8n-docs repository...'); + console.log('3ļøāƒ£ Discovering available nodes...'); + console.log('4ļøāƒ£ Extracting node information...\n'); + + const stats = await service.rebuildDatabase(); + + const duration = ((Date.now() - startTime) / 1000).toFixed(2); + + console.log('\nšŸ“Š Rebuild Results:'); + console.log(` Total nodes processed: ${stats.total}`); + console.log(` Successfully stored: ${stats.successful}`); + console.log(` Failed: ${stats.failed}`); + console.log(` Duration: ${duration}s`); + + if (stats.errors.length > 0) { + console.log('\nāš ļø First 5 errors:'); + stats.errors.slice(0, 5).forEach(error => { + console.log(` - ${error}`); + }); + if (stats.errors.length > 5) { + console.log(` ... and ${stats.errors.length - 5} more errors`); + } + } + + // Get final statistics + const dbStats = service.getStatistics(); + console.log('\nšŸ“ˆ Database Statistics:'); + console.log(` Total nodes: ${dbStats.totalNodes}`); + console.log(` Nodes with documentation: ${dbStats.nodesWithDocs}`); + console.log(` Nodes with examples: ${dbStats.nodesWithExamples}`); + console.log(` Trigger nodes: ${dbStats.triggerNodes}`); + console.log(` Webhook nodes: ${dbStats.webhookNodes}`); + console.log(` Total packages: ${dbStats.totalPackages}`); + + console.log('\nāœØ Database rebuild complete!'); + + } catch (error) { + console.error('\nāŒ Database rebuild failed:', error); + process.exit(1); + } finally { + service.close(); + } +} + +// Run if called directly +if (require.main === module) { + rebuildDatabase().catch(error => { + console.error(error); + process.exit(1); + }); +} + +export { rebuildDatabase }; \ No newline at end of file diff --git a/src/services/node-documentation-service.ts b/src/services/node-documentation-service.ts new file mode 100644 index 0000000..7e9b878 --- /dev/null +++ b/src/services/node-documentation-service.ts @@ -0,0 +1,547 @@ +import Database from 'better-sqlite3'; +import { createHash } from 'crypto'; +import path from 'path'; +import { promises as fs } from 'fs'; +import { logger } from '../utils/logger'; +import { NodeSourceExtractor } from '../utils/node-source-extractor'; +import { DocumentationFetcher } from '../utils/documentation-fetcher'; +import { ExampleGenerator } from '../utils/example-generator'; + +interface NodeInfo { + nodeType: string; + name: string; + displayName: string; + description: string; + category?: string; + subcategory?: string; + icon?: string; + sourceCode: string; + credentialCode?: string; + documentation?: string; + documentationUrl?: string; + exampleWorkflow?: any; + exampleParameters?: any; + propertiesSchema?: any; + packageName: string; + version?: string; + codexData?: any; + aliases?: string[]; + hasCredentials: boolean; + isTrigger: boolean; + isWebhook: boolean; +} + +interface SearchOptions { + query?: string; + nodeType?: string; + packageName?: string; + category?: string; + hasCredentials?: boolean; + isTrigger?: boolean; + limit?: number; +} + +export class NodeDocumentationService { + private db: Database.Database; + private extractor: NodeSourceExtractor; + private docsFetcher: DocumentationFetcher; + + constructor(dbPath?: string) { + const databasePath = dbPath || process.env.NODE_DB_PATH || path.join(process.cwd(), 'data', 'nodes.db'); + + // Ensure directory exists + const dbDir = path.dirname(databasePath); + if (!require('fs').existsSync(dbDir)) { + require('fs').mkdirSync(dbDir, { recursive: true }); + } + + this.db = new Database(databasePath); + this.extractor = new NodeSourceExtractor(); + this.docsFetcher = new DocumentationFetcher(); + + // Initialize database with new schema + this.initializeDatabase(); + + logger.info('Node Documentation Service initialized'); + } + + private initializeDatabase(): void { + // Execute the schema directly + const schema = ` +-- Main nodes table with documentation and examples +CREATE TABLE IF NOT EXISTS nodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + node_type TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + display_name TEXT, + description TEXT, + category TEXT, + subcategory TEXT, + icon TEXT, + + -- Source code + source_code TEXT NOT NULL, + credential_code TEXT, + code_hash TEXT NOT NULL, + code_length INTEGER NOT NULL, + + -- Documentation + documentation_markdown TEXT, + documentation_url TEXT, + + -- Example usage + example_workflow TEXT, + example_parameters TEXT, + properties_schema TEXT, + + -- Metadata + package_name TEXT NOT NULL, + version TEXT, + codex_data TEXT, + aliases TEXT, + + -- Flags + has_credentials INTEGER DEFAULT 0, + is_trigger INTEGER DEFAULT 0, + is_webhook INTEGER DEFAULT 0, + + -- Timestamps + extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_nodes_package_name ON nodes(package_name); +CREATE INDEX IF NOT EXISTS idx_nodes_category ON nodes(category); +CREATE INDEX IF NOT EXISTS idx_nodes_code_hash ON nodes(code_hash); +CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name); +CREATE INDEX IF NOT EXISTS idx_nodes_is_trigger ON nodes(is_trigger); + +-- Full Text Search +CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5( + node_type, + name, + display_name, + description, + category, + documentation_markdown, + aliases, + content=nodes, + content_rowid=id +); + +-- Triggers for FTS +CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes +BEGIN + INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases) + VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases); +END; + +CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes +BEGIN + DELETE FROM nodes_fts WHERE rowid = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes +BEGIN + DELETE FROM nodes_fts WHERE rowid = old.id; + INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, category, documentation_markdown, aliases) + VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.category, new.documentation_markdown, new.aliases); +END; + +-- Documentation sources table +CREATE TABLE IF NOT EXISTS documentation_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + commit_hash TEXT, + fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Statistics table +CREATE TABLE IF NOT EXISTS extraction_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + total_nodes INTEGER NOT NULL, + nodes_with_docs INTEGER NOT NULL, + nodes_with_examples INTEGER NOT NULL, + total_code_size INTEGER NOT NULL, + total_docs_size INTEGER NOT NULL, + extraction_date DATETIME DEFAULT CURRENT_TIMESTAMP +); + `; + + this.db.exec(schema); + } + + /** + * Store complete node information including docs and examples + */ + async storeNode(nodeInfo: NodeInfo): Promise { + const hash = this.generateHash(nodeInfo.sourceCode); + + const stmt = this.db.prepare(` + INSERT OR REPLACE INTO nodes ( + node_type, name, display_name, description, category, subcategory, icon, + source_code, credential_code, code_hash, code_length, + documentation_markdown, documentation_url, + example_workflow, example_parameters, properties_schema, + package_name, version, codex_data, aliases, + has_credentials, is_trigger, is_webhook + ) VALUES ( + @nodeType, @name, @displayName, @description, @category, @subcategory, @icon, + @sourceCode, @credentialCode, @hash, @codeLength, + @documentation, @documentationUrl, + @exampleWorkflow, @exampleParameters, @propertiesSchema, + @packageName, @version, @codexData, @aliases, + @hasCredentials, @isTrigger, @isWebhook + ) + `); + + stmt.run({ + nodeType: nodeInfo.nodeType, + name: nodeInfo.name, + displayName: nodeInfo.displayName || nodeInfo.name, + description: nodeInfo.description || '', + category: nodeInfo.category || 'Other', + subcategory: nodeInfo.subcategory || null, + icon: nodeInfo.icon || null, + sourceCode: nodeInfo.sourceCode, + credentialCode: nodeInfo.credentialCode || null, + hash, + codeLength: nodeInfo.sourceCode.length, + documentation: nodeInfo.documentation || null, + documentationUrl: nodeInfo.documentationUrl || null, + exampleWorkflow: nodeInfo.exampleWorkflow ? JSON.stringify(nodeInfo.exampleWorkflow) : null, + exampleParameters: nodeInfo.exampleParameters ? JSON.stringify(nodeInfo.exampleParameters) : null, + propertiesSchema: nodeInfo.propertiesSchema ? JSON.stringify(nodeInfo.propertiesSchema) : null, + packageName: nodeInfo.packageName, + version: nodeInfo.version || null, + codexData: nodeInfo.codexData ? JSON.stringify(nodeInfo.codexData) : null, + aliases: nodeInfo.aliases ? JSON.stringify(nodeInfo.aliases) : null, + hasCredentials: nodeInfo.hasCredentials ? 1 : 0, + isTrigger: nodeInfo.isTrigger ? 1 : 0, + isWebhook: nodeInfo.isWebhook ? 1 : 0 + }); + } + + /** + * Get complete node information + */ + async getNodeInfo(nodeType: string): Promise { + const stmt = this.db.prepare(` + SELECT * FROM nodes WHERE node_type = ? OR name = ? COLLATE NOCASE + `); + + const row = stmt.get(nodeType, nodeType); + if (!row) return null; + + return this.rowToNodeInfo(row); + } + + /** + * Search nodes with various filters + */ + async searchNodes(options: SearchOptions): Promise { + let query = 'SELECT * FROM nodes WHERE 1=1'; + const params: any = {}; + + if (options.query) { + query += ` AND id IN ( + SELECT rowid FROM nodes_fts + WHERE nodes_fts MATCH @query + )`; + params.query = options.query; + } + + if (options.nodeType) { + query += ' AND node_type LIKE @nodeType'; + params.nodeType = `%${options.nodeType}%`; + } + + if (options.packageName) { + query += ' AND package_name = @packageName'; + params.packageName = options.packageName; + } + + if (options.category) { + query += ' AND category = @category'; + params.category = options.category; + } + + if (options.hasCredentials !== undefined) { + query += ' AND has_credentials = @hasCredentials'; + params.hasCredentials = options.hasCredentials ? 1 : 0; + } + + if (options.isTrigger !== undefined) { + query += ' AND is_trigger = @isTrigger'; + params.isTrigger = options.isTrigger ? 1 : 0; + } + + query += ' ORDER BY name LIMIT @limit'; + params.limit = options.limit || 20; + + const stmt = this.db.prepare(query); + const rows = stmt.all(params); + + return rows.map(row => this.rowToNodeInfo(row)); + } + + /** + * List all nodes + */ + async listNodes(): Promise { + const stmt = this.db.prepare('SELECT * FROM nodes ORDER BY name'); + const rows = stmt.all(); + return rows.map(row => this.rowToNodeInfo(row)); + } + + /** + * Extract and store all nodes with documentation + */ + async rebuildDatabase(): Promise<{ + total: number; + successful: number; + failed: number; + errors: string[]; + }> { + logger.info('Starting complete database rebuild...'); + + // Clear existing data + this.db.exec('DELETE FROM nodes'); + this.db.exec('DELETE FROM extraction_stats'); + + // Ensure documentation repository is available + await this.docsFetcher.ensureDocsRepository(); + + const stats = { + total: 0, + successful: 0, + failed: 0, + errors: [] as string[] + }; + + try { + // Get all available nodes + const availableNodes = await this.extractor.listAvailableNodes(); + stats.total = availableNodes.length; + + logger.info(`Found ${stats.total} nodes to process`); + + // Process nodes in batches + const batchSize = 10; + for (let i = 0; i < availableNodes.length; i += batchSize) { + const batch = availableNodes.slice(i, i + batchSize); + + await Promise.all(batch.map(async (node) => { + try { + // Build node type from package name and node name + const nodeType = `n8n-nodes-base.${node.name}`; + + // Extract source code + const nodeData = await this.extractor.extractNodeSource(nodeType); + if (!nodeData || !nodeData.sourceCode) { + throw new Error('Failed to extract node source'); + } + + // Parse node definition to get metadata + const nodeDefinition = this.parseNodeDefinition(nodeData.sourceCode); + + // Get documentation + const docs = await this.docsFetcher.getNodeDocumentation(nodeType); + + // Generate example + const example = ExampleGenerator.generateFromNodeDefinition(nodeDefinition); + + // Prepare node info + const nodeInfo: NodeInfo = { + nodeType: nodeType, + name: node.name, + displayName: nodeDefinition.displayName || node.displayName || node.name, + description: nodeDefinition.description || node.description || '', + category: nodeDefinition.category || 'Other', + subcategory: nodeDefinition.subcategory, + icon: nodeDefinition.icon, + sourceCode: nodeData.sourceCode, + credentialCode: nodeData.credentialCode, + documentation: docs?.markdown, + documentationUrl: docs?.url, + exampleWorkflow: example, + exampleParameters: example.nodes[0]?.parameters, + propertiesSchema: nodeDefinition.properties, + packageName: nodeData.packageInfo?.name || 'n8n-nodes-base', + version: nodeDefinition.version, + codexData: nodeDefinition.codex, + aliases: nodeDefinition.alias, + hasCredentials: !!nodeData.credentialCode, + isTrigger: node.name.toLowerCase().includes('trigger'), + isWebhook: node.name.toLowerCase().includes('webhook') + }; + + // Store in database + await this.storeNode(nodeInfo); + + stats.successful++; + logger.debug(`Processed node: ${nodeType}`); + } catch (error) { + stats.failed++; + const errorMsg = `Failed to process ${node.name}: ${error instanceof Error ? error.message : String(error)}`; + stats.errors.push(errorMsg); + logger.error(errorMsg); + } + })); + + logger.info(`Progress: ${Math.min(i + batchSize, availableNodes.length)}/${stats.total} nodes processed`); + } + + // Store statistics + this.storeStatistics(stats); + + logger.info(`Database rebuild complete: ${stats.successful} successful, ${stats.failed} failed`); + + } catch (error) { + logger.error('Database rebuild failed:', error); + throw error; + } + + return stats; + } + + /** + * Parse node definition from source code + */ + private parseNodeDefinition(sourceCode: string): any { + try { + // Try to extract the description object from the source + const descMatch = sourceCode.match(/description\s*[:=]\s*({[\s\S]*?})\s*[,;]/); + if (descMatch) { + // Clean up the match and try to parse it + const descStr = descMatch[1] + .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":') // Quote property names + .replace(/:\s*'([^']*)'/g, ': "$1"') // Convert single quotes to double + .replace(/,\s*}/g, '}'); // Remove trailing commas + + return JSON.parse(descStr); + } + } catch (error) { + logger.debug('Failed to parse node definition:', error); + } + + // Return minimal definition if parsing fails + return { + displayName: '', + description: '', + properties: [] + }; + } + + /** + * Convert database row to NodeInfo + */ + private rowToNodeInfo(row: any): NodeInfo { + return { + nodeType: row.node_type, + name: row.name, + displayName: row.display_name, + description: row.description, + category: row.category, + subcategory: row.subcategory, + icon: row.icon, + sourceCode: row.source_code, + credentialCode: row.credential_code, + documentation: row.documentation_markdown, + documentationUrl: row.documentation_url, + exampleWorkflow: row.example_workflow ? JSON.parse(row.example_workflow) : null, + exampleParameters: row.example_parameters ? JSON.parse(row.example_parameters) : null, + propertiesSchema: row.properties_schema ? JSON.parse(row.properties_schema) : null, + packageName: row.package_name, + version: row.version, + codexData: row.codex_data ? JSON.parse(row.codex_data) : null, + aliases: row.aliases ? JSON.parse(row.aliases) : null, + hasCredentials: row.has_credentials === 1, + isTrigger: row.is_trigger === 1, + isWebhook: row.is_webhook === 1 + }; + } + + /** + * Generate hash for content + */ + private generateHash(content: string): string { + return createHash('sha256').update(content).digest('hex'); + } + + /** + * Store extraction statistics + */ + private storeStatistics(stats: any): void { + const stmt = this.db.prepare(` + INSERT INTO extraction_stats ( + total_nodes, nodes_with_docs, nodes_with_examples, + total_code_size, total_docs_size + ) VALUES (?, ?, ?, ?, ?) + `); + + // Calculate sizes + const sizeStats = this.db.prepare(` + SELECT + COUNT(*) as total, + SUM(CASE WHEN documentation_markdown IS NOT NULL THEN 1 ELSE 0 END) as with_docs, + SUM(CASE WHEN example_workflow IS NOT NULL THEN 1 ELSE 0 END) as with_examples, + SUM(code_length) as code_size, + SUM(LENGTH(documentation_markdown)) as docs_size + FROM nodes + `).get() as any; + + stmt.run( + stats.successful, + sizeStats?.with_docs || 0, + sizeStats?.with_examples || 0, + sizeStats?.code_size || 0, + sizeStats?.docs_size || 0 + ); + } + + /** + * Get database statistics + */ + getStatistics(): any { + const stats = this.db.prepare(` + SELECT + COUNT(*) as totalNodes, + COUNT(DISTINCT package_name) as totalPackages, + SUM(code_length) as totalCodeSize, + SUM(CASE WHEN documentation_markdown IS NOT NULL THEN 1 ELSE 0 END) as nodesWithDocs, + SUM(CASE WHEN example_workflow IS NOT NULL THEN 1 ELSE 0 END) as nodesWithExamples, + SUM(has_credentials) as nodesWithCredentials, + SUM(is_trigger) as triggerNodes, + SUM(is_webhook) as webhookNodes + FROM nodes + `).get() as any; + + const packages = this.db.prepare(` + SELECT package_name as package, COUNT(*) as count + FROM nodes + GROUP BY package_name + ORDER BY count DESC + `).all(); + + return { + totalNodes: stats?.totalNodes || 0, + totalPackages: stats?.totalPackages || 0, + totalCodeSize: stats?.totalCodeSize || 0, + nodesWithDocs: stats?.nodesWithDocs || 0, + nodesWithExamples: stats?.nodesWithExamples || 0, + nodesWithCredentials: stats?.nodesWithCredentials || 0, + triggerNodes: stats?.triggerNodes || 0, + webhookNodes: stats?.webhookNodes || 0, + packageDistribution: packages + }; + } + + /** + * Close database connection + */ + close(): void { + this.db.close(); + } +} \ No newline at end of file diff --git a/src/utils/documentation-fetcher.ts b/src/utils/documentation-fetcher.ts new file mode 100644 index 0000000..2ce29ea --- /dev/null +++ b/src/utils/documentation-fetcher.ts @@ -0,0 +1,241 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { logger } from './logger'; +import { execSync } from 'child_process'; + +interface NodeDocumentation { + markdown: string; + url: string; + examples?: any[]; +} + +export class DocumentationFetcher { + private docsPath: string; + private docsRepoUrl = 'https://github.com/n8n-io/n8n-docs.git'; + private cloned = false; + + constructor(docsPath?: string) { + this.docsPath = docsPath || path.join(process.cwd(), 'temp', 'n8n-docs'); + } + + /** + * Clone or update the n8n-docs repository + */ + async ensureDocsRepository(): Promise { + try { + // Check if directory exists + const exists = await fs.access(this.docsPath).then(() => true).catch(() => false); + + if (!exists) { + logger.info('Cloning n8n-docs repository...'); + await fs.mkdir(path.dirname(this.docsPath), { recursive: true }); + execSync(`git clone --depth 1 ${this.docsRepoUrl} ${this.docsPath}`, { + stdio: 'pipe' + }); + logger.info('n8n-docs repository cloned successfully'); + } else { + logger.info('Updating n8n-docs repository...'); + execSync('git pull --ff-only', { + cwd: this.docsPath, + stdio: 'pipe' + }); + logger.info('n8n-docs repository updated'); + } + + this.cloned = true; + } catch (error) { + logger.error('Failed to clone/update n8n-docs repository:', error); + throw error; + } + } + + /** + * Get documentation for a specific node + */ + async getNodeDocumentation(nodeType: string): Promise { + if (!this.cloned) { + await this.ensureDocsRepository(); + } + + try { + // Convert node type to documentation path + // e.g., "n8n-nodes-base.if" -> "if" + const nodeName = this.extractNodeName(nodeType); + + // Common documentation paths to check + const possiblePaths = [ + path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'core-nodes', `${nodeName}.md`), + path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'app-nodes', `${nodeName}.md`), + path.join(this.docsPath, 'docs', 'integrations', 'builtin', 'trigger-nodes', `${nodeName}.md`), + path.join(this.docsPath, 'docs', 'code-examples', 'expressions', `${nodeName}.md`), + // Generic search in docs folder + path.join(this.docsPath, 'docs', '**', `${nodeName}.md`) + ]; + + for (const docPath of possiblePaths) { + try { + const content = await fs.readFile(docPath, 'utf-8'); + const url = this.generateDocUrl(docPath); + + return { + markdown: content, + url, + examples: this.extractExamples(content) + }; + } catch (error) { + // Continue to next path + continue; + } + } + + // If no exact match, try to find by searching + const foundPath = await this.searchForNodeDoc(nodeName); + if (foundPath) { + const content = await fs.readFile(foundPath, 'utf-8'); + return { + markdown: content, + url: this.generateDocUrl(foundPath), + examples: this.extractExamples(content) + }; + } + + logger.warn(`No documentation found for node: ${nodeType}`); + return null; + } catch (error) { + logger.error(`Failed to get documentation for ${nodeType}:`, error); + return null; + } + } + + /** + * Extract node name from node type + */ + private extractNodeName(nodeType: string): string { + // Handle different node type formats + // "n8n-nodes-base.if" -> "if" + // "@n8n/n8n-nodes-langchain.Agent" -> "agent" + const parts = nodeType.split('.'); + const name = parts[parts.length - 1]; + return name.toLowerCase(); + } + + /** + * Search for node documentation file + */ + private async searchForNodeDoc(nodeName: string): Promise { + try { + const result = execSync( + `find ${this.docsPath}/docs -name "*.md" -type f | grep -i "${nodeName}" | head -1`, + { encoding: 'utf-8', stdio: 'pipe' } + ).trim(); + + return result || null; + } catch (error) { + return null; + } + } + + /** + * Generate documentation URL from file path + */ + private generateDocUrl(filePath: string): string { + const relativePath = path.relative(this.docsPath, filePath); + const urlPath = relativePath + .replace(/^docs\//, '') + .replace(/\.md$/, '') + .replace(/\\/g, '/'); + + return `https://docs.n8n.io/${urlPath}`; + } + + /** + * Extract code examples from markdown content + */ + private extractExamples(markdown: string): any[] { + const examples: any[] = []; + + // Extract JSON code blocks + const jsonCodeBlockRegex = /```json\n([\s\S]*?)```/g; + let match; + + while ((match = jsonCodeBlockRegex.exec(markdown)) !== null) { + try { + const json = JSON.parse(match[1]); + examples.push(json); + } catch (error) { + // Not valid JSON, skip + } + } + + // Extract workflow examples + const workflowExampleRegex = /## Example.*?\n([\s\S]*?)(?=\n##|\n#|$)/gi; + while ((match = workflowExampleRegex.exec(markdown)) !== null) { + const exampleText = match[1]; + // Try to find JSON in the example section + const jsonMatch = exampleText.match(/```json\n([\s\S]*?)```/); + if (jsonMatch) { + try { + const json = JSON.parse(jsonMatch[1]); + examples.push(json); + } catch (error) { + // Not valid JSON + } + } + } + + return examples; + } + + /** + * Get all available documentation files + */ + async getAllDocumentationFiles(): Promise> { + if (!this.cloned) { + await this.ensureDocsRepository(); + } + + const docMap = new Map(); + + try { + const findDocs = execSync( + `find ${this.docsPath}/docs -name "*.md" -type f | grep -E "(core-nodes|app-nodes|trigger-nodes)/"`, + { encoding: 'utf-8', stdio: 'pipe' } + ).trim().split('\n'); + + for (const docPath of findDocs) { + if (!docPath) continue; + + const filename = path.basename(docPath, '.md'); + const content = await fs.readFile(docPath, 'utf-8'); + + // Try to extract the node type from the content + const nodeTypeMatch = content.match(/node[_-]?type[:\s]+["']?([^"'\s]+)["']?/i); + if (nodeTypeMatch) { + docMap.set(nodeTypeMatch[1], docPath); + } else { + // Use filename as fallback + docMap.set(filename, docPath); + } + } + + logger.info(`Found ${docMap.size} documentation files`); + return docMap; + } catch (error) { + logger.error('Failed to get documentation files:', error); + return docMap; + } + } + + /** + * Clean up cloned repository + */ + async cleanup(): Promise { + try { + await fs.rm(this.docsPath, { recursive: true, force: true }); + this.cloned = false; + logger.info('Cleaned up documentation repository'); + } catch (error) { + logger.error('Failed to cleanup docs repository:', error); + } + } +} \ No newline at end of file diff --git a/src/utils/example-generator.ts b/src/utils/example-generator.ts new file mode 100644 index 0000000..6469a24 --- /dev/null +++ b/src/utils/example-generator.ts @@ -0,0 +1,267 @@ +import { logger } from './logger'; + +interface NodeExample { + nodes: any[]; + connections: any; + pinData?: any; + meta?: any; +} + +interface NodeParameter { + name: string; + type: string; + default?: any; + options?: any[]; + displayOptions?: any; +} + +export class ExampleGenerator { + /** + * Generate example workflow for a node + */ + static generateNodeExample(nodeType: string, nodeData: any): NodeExample { + const nodeName = this.getNodeName(nodeType); + const nodeId = this.generateNodeId(); + + // Base example structure + const example: NodeExample = { + nodes: [{ + parameters: this.generateExampleParameters(nodeType, nodeData), + type: nodeType, + typeVersion: nodeData.typeVersion || 1, + position: [220, 120], + id: nodeId, + name: nodeName + }], + connections: { + [nodeName]: { + main: [[]] + } + }, + pinData: {}, + meta: { + templateCredsSetupCompleted: true, + instanceId: this.generateInstanceId() + } + }; + + // Add specific configurations based on node type + this.addNodeSpecificConfig(nodeType, example, nodeData); + + return example; + } + + /** + * Generate example parameters based on node type + */ + private static generateExampleParameters(nodeType: string, nodeData: any): any { + const params: any = {}; + + // Extract node name for specific handling + const nodeName = nodeType.split('.').pop()?.toLowerCase() || ''; + + // Common node examples + switch (nodeName) { + case 'if': + return { + conditions: { + options: { + caseSensitive: true, + leftValue: "", + typeValidation: "strict", + version: 2 + }, + conditions: [{ + id: this.generateNodeId(), + leftValue: "={{ $json }}", + rightValue: "", + operator: { + type: "object", + operation: "notEmpty", + singleValue: true + } + }], + combinator: "and" + }, + options: {} + }; + + case 'webhook': + return { + httpMethod: "POST", + path: "webhook-path", + responseMode: "onReceived", + responseData: "allEntries", + options: {} + }; + + case 'httprequest': + return { + method: "GET", + url: "https://api.example.com/data", + authentication: "none", + options: {}, + headerParametersUi: { + parameter: [] + } + }; + + case 'function': + return { + functionCode: "// Add your JavaScript code here\nreturn $input.all();" + }; + + case 'set': + return { + mode: "manual", + duplicateItem: false, + values: { + string: [{ + name: "myField", + value: "myValue" + }] + } + }; + + case 'split': + return { + batchSize: 10, + options: {} + }; + + default: + // Generate generic parameters from node properties + return this.generateGenericParameters(nodeData); + } + } + + /** + * Generate generic parameters from node properties + */ + private static generateGenericParameters(nodeData: any): any { + const params: any = {}; + + if (nodeData.properties) { + for (const prop of nodeData.properties) { + if (prop.default !== undefined) { + params[prop.name] = prop.default; + } else if (prop.type === 'string') { + params[prop.name] = ''; + } else if (prop.type === 'number') { + params[prop.name] = 0; + } else if (prop.type === 'boolean') { + params[prop.name] = false; + } else if (prop.type === 'options' && prop.options?.length > 0) { + params[prop.name] = prop.options[0].value; + } + } + } + + return params; + } + + /** + * Add node-specific configurations + */ + private static addNodeSpecificConfig(nodeType: string, example: NodeExample, nodeData: any): void { + const nodeName = nodeType.split('.').pop()?.toLowerCase() || ''; + + // Add specific connection structures for different node types + switch (nodeName) { + case 'if': + // IF node has true/false outputs + example.connections[example.nodes[0].name] = { + main: [[], []] // Two outputs: true, false + }; + break; + + case 'switch': + // Switch node can have multiple outputs + const outputs = nodeData.outputs || 3; + example.connections[example.nodes[0].name] = { + main: Array(outputs).fill([]) + }; + break; + + case 'merge': + // Merge node has multiple inputs + example.nodes[0].position = [400, 120]; + // Add dummy input nodes + example.nodes.push({ + parameters: {}, + type: "n8n-nodes-base.noOp", + typeVersion: 1, + position: [200, 60], + id: this.generateNodeId(), + name: "Input 1" + }); + example.nodes.push({ + parameters: {}, + type: "n8n-nodes-base.noOp", + typeVersion: 1, + position: [200, 180], + id: this.generateNodeId(), + name: "Input 2" + }); + example.connections = { + "Input 1": { main: [[{ node: example.nodes[0].name, type: "main", index: 0 }]] }, + "Input 2": { main: [[{ node: example.nodes[0].name, type: "main", index: 1 }]] }, + [example.nodes[0].name]: { main: [[]] } + }; + break; + } + + // Add credentials if needed + if (nodeData.credentials?.length > 0) { + example.nodes[0].credentials = {}; + for (const cred of nodeData.credentials) { + example.nodes[0].credentials[cred.name] = { + id: this.generateNodeId(), + name: `${cred.name} account` + }; + } + } + } + + /** + * Extract display name from node type + */ + private static getNodeName(nodeType: string): string { + const parts = nodeType.split('.'); + const name = parts[parts.length - 1]; + return name.charAt(0).toUpperCase() + name.slice(1); + } + + /** + * Generate a random node ID + */ + private static generateNodeId(): string { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { + const r = Math.random() * 16 | 0; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } + + /** + * Generate instance ID + */ + private static generateInstanceId(): string { + return Array(64).fill(0).map(() => Math.floor(Math.random() * 16).toString(16)).join(''); + } + + /** + * Generate example from node definition + */ + static generateFromNodeDefinition(nodeDefinition: any): NodeExample { + const nodeType = nodeDefinition.description?.name || 'n8n-nodes-base.node'; + const nodeData = { + typeVersion: nodeDefinition.description?.version || 1, + properties: nodeDefinition.description?.properties || [], + credentials: nodeDefinition.description?.credentials || [], + outputs: nodeDefinition.description?.outputs || ['main'] + }; + + return this.generateNodeExample(nodeType, nodeData); + } +} \ No newline at end of file diff --git a/tests/test-node-documentation-service.js b/tests/test-node-documentation-service.js new file mode 100644 index 0000000..44a3e3a --- /dev/null +++ b/tests/test-node-documentation-service.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node + +const { NodeDocumentationService } = require('../dist/services/node-documentation-service'); + +async function testService() { + console.log('=== Testing Node Documentation Service ===\n'); + + // Use a separate database for v2 + const service = new NodeDocumentationService('./data/nodes-v2.db'); + + try { + // Test 1: List nodes + console.log('1ļøāƒ£ Testing list nodes...'); + const nodes = await service.listNodes(); + console.log(` Found ${nodes.length} nodes in database`); + + if (nodes.length === 0) { + console.log('\nāš ļø No nodes found. Running rebuild...'); + const stats = await service.rebuildDatabase(); + console.log(` Rebuild complete: ${stats.successful} nodes stored`); + } + + // Test 2: Get specific node info (IF node) + console.log('\n2ļøāƒ£ Testing get node info for "If" node...'); + const ifNode = await service.getNodeInfo('n8n-nodes-base.if'); + + if (ifNode) { + console.log(' āœ… Found IF node:'); + console.log(` Name: ${ifNode.displayName}`); + console.log(` Description: ${ifNode.description}`); + console.log(` Has source code: ${!!ifNode.sourceCode}`); + console.log(` Source code length: ${ifNode.sourceCode?.length || 0} bytes`); + console.log(` Has documentation: ${!!ifNode.documentation}`); + console.log(` Has example: ${!!ifNode.exampleWorkflow}`); + + if (ifNode.exampleWorkflow) { + console.log('\n šŸ“‹ Example workflow:'); + console.log(JSON.stringify(ifNode.exampleWorkflow, null, 2).substring(0, 500) + '...'); + } + } else { + console.log(' āŒ IF node not found'); + } + + // Test 3: Search nodes + console.log('\n3ļøāƒ£ Testing search functionality...'); + + // Search for webhook nodes + const webhookNodes = await service.searchNodes({ query: 'webhook' }); + console.log(`\n šŸ” Search for "webhook": ${webhookNodes.length} results`); + webhookNodes.slice(0, 3).forEach(node => { + console.log(` - ${node.displayName} (${node.nodeType})`); + }); + + // Search for HTTP nodes + const httpNodes = await service.searchNodes({ query: 'http' }); + console.log(`\n šŸ” Search for "http": ${httpNodes.length} results`); + httpNodes.slice(0, 3).forEach(node => { + console.log(` - ${node.displayName} (${node.nodeType})`); + }); + + // Test 4: Get statistics + console.log('\n4ļøāƒ£ Testing database statistics...'); + const stats = service.getStatistics(); + console.log(' šŸ“Š Database stats:'); + console.log(` Total nodes: ${stats.totalNodes}`); + console.log(` Nodes with docs: ${stats.nodesWithDocs}`); + console.log(` Nodes with examples: ${stats.nodesWithExamples}`); + console.log(` Trigger nodes: ${stats.triggerNodes}`); + console.log(` Webhook nodes: ${stats.webhookNodes}`); + console.log(` Total packages: ${stats.totalPackages}`); + + // Test 5: Category filtering + console.log('\n5ļøāƒ£ Testing category filtering...'); + const coreNodes = await service.searchNodes({ category: 'Core Nodes' }); + console.log(` Found ${coreNodes.length} core nodes`); + + console.log('\nāœ… All tests completed!'); + + } catch (error) { + console.error('\nāŒ Test failed:', error); + process.exit(1); + } finally { + service.close(); + } +} + +// Run tests +testService().catch(console.error); \ No newline at end of file diff --git a/tests/test-node-list.js b/tests/test-node-list.js new file mode 100644 index 0000000..e95eba7 --- /dev/null +++ b/tests/test-node-list.js @@ -0,0 +1,26 @@ +#!/usr/bin/env node + +const { NodeSourceExtractor } = require('../dist/utils/node-source-extractor'); + +async function testNodeList() { + console.log('Testing node list...\n'); + + const extractor = new NodeSourceExtractor(); + + try { + const nodes = await extractor.listAvailableNodes(); + + console.log(`Total nodes found: ${nodes.length}`); + + // Show first 5 nodes + console.log('\nFirst 5 nodes:'); + nodes.slice(0, 5).forEach((node, index) => { + console.log(`${index + 1}. Node:`, JSON.stringify(node, null, 2)); + }); + + } catch (error) { + console.error('Error:', error); + } +} + +testNodeList(); \ No newline at end of file diff --git a/tests/test-small-rebuild.js b/tests/test-small-rebuild.js new file mode 100644 index 0000000..15450cb --- /dev/null +++ b/tests/test-small-rebuild.js @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +const { NodeDocumentationService } = require('../dist/services/node-documentation-service'); + +async function testSmallRebuild() { + console.log('Testing small rebuild...\n'); + + const service = new NodeDocumentationService('./data/nodes-v2-test.db'); + + try { + // First, let's just try the IF node specifically + const extractor = service.extractor; + console.log('1ļøāƒ£ Testing extraction of IF node...'); + + try { + const ifNodeData = await extractor.extractNodeSource('n8n-nodes-base.If'); + console.log(' āœ… Successfully extracted IF node'); + console.log(' Source code length:', ifNodeData.sourceCode.length); + console.log(' Has credentials:', !!ifNodeData.credentialCode); + } catch (error) { + console.log(' āŒ Failed to extract IF node:', error.message); + } + + // Try the Webhook node + console.log('\n2ļøāƒ£ Testing extraction of Webhook node...'); + try { + const webhookNodeData = await extractor.extractNodeSource('n8n-nodes-base.Webhook'); + console.log(' āœ… Successfully extracted Webhook node'); + console.log(' Source code length:', webhookNodeData.sourceCode.length); + } catch (error) { + console.log(' āŒ Failed to extract Webhook node:', error.message); + } + + // Now try storing just these nodes + console.log('\n3ļøāƒ£ Testing storage of a single node...'); + const nodeInfo = { + nodeType: 'n8n-nodes-base.If', + name: 'If', + displayName: 'If', + description: 'Route items based on comparison operations', + sourceCode: 'test source code', + packageName: 'n8n-nodes-base', + hasCredentials: false, + isTrigger: false, + isWebhook: false + }; + + await service.storeNode(nodeInfo); + console.log(' āœ… Successfully stored test node'); + + // Check if it was stored + const retrievedNode = await service.getNodeInfo('n8n-nodes-base.If'); + console.log(' Retrieved node:', retrievedNode ? 'Found' : 'Not found'); + + } catch (error) { + console.error('āŒ Test failed:', error); + } finally { + service.close(); + } +} + +testSmallRebuild(); \ No newline at end of file