feat: add node extraction scripts and Docker setup for n8n modules

This commit is contained in:
czlonkowski
2025-06-09 00:33:17 +02:00
parent 3d7fdeba02
commit 6851d2f253
20 changed files with 737 additions and 1 deletions

View File

@@ -15,6 +15,7 @@ RUN npm ci --only=production
# Copy built files
COPY dist ./dist
COPY tests ./tests
COPY scripts ./scripts
# Create data directory for SQLite database
RUN mkdir -p /app/data

BIN
data/demo-enhanced.db-shm Normal file

Binary file not shown.

View File

BIN
data/nodes-backup.db Normal file

Binary file not shown.

BIN
data/nodes-enhanced.db Normal file

Binary file not shown.

BIN
data/nodes-fresh.db Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
data/test-enhanced.db-shm Normal file

Binary file not shown.

View File

View File

@@ -0,0 +1,49 @@
version: '3.8'
services:
# Latest n8n container for node extraction
n8n-latest:
image: n8nio/n8n:latest
container_name: n8n-latest-extractor
environment:
- N8N_BASIC_AUTH_ACTIVE=false
- N8N_PORT=5678
- N8N_ENCRYPTION_KEY=dummy-key-for-extraction
volumes:
# Mount n8n's node_modules for extraction
- n8n_modules:/usr/local/lib/node_modules/n8n/node_modules
# Provide writable directory for n8n config
- n8n_config:/home/node/.n8n
# We don't need n8n to stay running, just to install modules
entrypoint: ["sh", "-c", "sleep 300"]
healthcheck:
test: ["CMD-SHELL", "ls /usr/local/lib/node_modules/n8n/node_modules/n8n-nodes-base > /dev/null 2>&1"]
interval: 5s
timeout: 30s
retries: 20
# Extractor service that will read from the mounted volumes
node-extractor:
image: node:18-alpine
container_name: n8n-node-extractor
working_dir: /app
depends_on:
n8n-latest:
condition: service_healthy
volumes:
# Mount the n8n modules from the n8n container
- n8n_modules:/n8n-modules:ro
- n8n_custom:/n8n-custom:ro
# Mount our project files
- ./:/app
environment:
- NODE_ENV=development
- NODE_DB_PATH=/app/data/nodes-fresh.db
- N8N_MODULES_PATH=/n8n-modules
- N8N_CUSTOM_PATH=/n8n-custom
command: /bin/sh -c "apk add --no-cache sqlite && node /app/scripts/extract-from-docker.js"
volumes:
n8n_modules:
n8n_custom:
n8n_config:

View File

@@ -0,0 +1,18 @@
{
"mcpServers": {
"puppeteer": {
"command": "npx",
"args": [
"-y",
"@modelcontextprotocol/server-puppeteer"
]
},
"n8n-nodes": {
"command": "node",
"args": ["/Users/romualdczlonkowski/Pliki/n8n-mcp/n8n-mcp/dist/index-v2.js"],
"env": {
"NODE_DB_PATH": "/Users/romualdczlonkowski/Pliki/n8n-mcp/n8n-mcp/data/nodes-v2.db"
}
}
}
}

2
package-lock.json generated
View File

@@ -7,7 +7,7 @@
"": {
"name": "n8n-mcp",
"version": "1.0.0",
"license": "ISC",
"license": "Sustainable-Use-1.0",
"dependencies": {
"@modelcontextprotocol/sdk": "^1.12.1",
"@types/better-sqlite3": "^7.6.13",

View File

@@ -0,0 +1,220 @@
#!/usr/bin/env node
const dotenv = require('dotenv');
const { NodeDocumentationService } = require('../dist/services/node-documentation-service');
const { NodeSourceExtractor } = require('../dist/utils/node-source-extractor');
const { logger } = require('../dist/utils/logger');
const fs = require('fs').promises;
const path = require('path');
// Load environment variables
dotenv.config();
async function extractNodesFromDocker() {
logger.info('🐳 Starting Docker-based node extraction...');
// Add Docker volume paths to environment for NodeSourceExtractor
const dockerVolumePaths = [
process.env.N8N_MODULES_PATH || '/n8n-modules',
process.env.N8N_CUSTOM_PATH || '/n8n-custom',
];
logger.info(`Docker volume paths: ${dockerVolumePaths.join(', ')}`);
// Check if volumes are mounted
for (const volumePath of dockerVolumePaths) {
try {
await fs.access(volumePath);
logger.info(`✅ Volume mounted: ${volumePath}`);
// List what's in the volume
const entries = await fs.readdir(volumePath);
logger.info(`Contents of ${volumePath}: ${entries.slice(0, 10).join(', ')}${entries.length > 10 ? '...' : ''}`);
} catch (error) {
logger.warn(`❌ Volume not accessible: ${volumePath}`);
}
}
// Initialize services
const docService = new NodeDocumentationService();
const extractor = new NodeSourceExtractor();
// Extend the extractor's search paths with Docker volumes
extractor.n8nBasePaths.unshift(...dockerVolumePaths);
// Clear existing nodes to ensure we only have latest versions
logger.info('🧹 Clearing existing nodes...');
const db = docService.db;
db.prepare('DELETE FROM nodes').run();
logger.info('🔍 Searching for n8n nodes in Docker volumes...');
// Known n8n packages to extract
const n8nPackages = [
'n8n-nodes-base',
'@n8n/n8n-nodes-langchain',
'n8n-nodes-extras',
];
let totalExtracted = 0;
let ifNodeVersion = null;
for (const packageName of n8nPackages) {
logger.info(`\n📦 Processing package: ${packageName}`);
try {
// Find package in Docker volumes
let packagePath = null;
for (const volumePath of dockerVolumePaths) {
const possiblePaths = [
path.join(volumePath, packageName),
path.join(volumePath, '.pnpm', `${packageName}@*`, 'node_modules', packageName),
];
for (const testPath of possiblePaths) {
try {
// Use glob pattern to find pnpm packages
if (testPath.includes('*')) {
const baseDir = path.dirname(testPath.split('*')[0]);
const entries = await fs.readdir(baseDir);
for (const entry of entries) {
if (entry.includes(packageName.replace('/', '+'))) {
const fullPath = path.join(baseDir, entry, 'node_modules', packageName);
try {
await fs.access(fullPath);
packagePath = fullPath;
break;
} catch {}
}
}
} else {
await fs.access(testPath);
packagePath = testPath;
break;
}
} catch {}
}
if (packagePath) break;
}
if (!packagePath) {
logger.warn(`Package ${packageName} not found in Docker volumes`);
continue;
}
logger.info(`Found package at: ${packagePath}`);
// Check package version
try {
const packageJsonPath = path.join(packagePath, 'package.json');
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf-8'));
logger.info(`Package version: ${packageJson.version}`);
} catch {}
// Find nodes directory
const nodesPath = path.join(packagePath, 'dist', 'nodes');
try {
await fs.access(nodesPath);
logger.info(`Scanning nodes directory: ${nodesPath}`);
// Extract all nodes from this package
const nodeEntries = await scanForNodes(nodesPath);
logger.info(`Found ${nodeEntries.length} nodes in ${packageName}`);
for (const nodeEntry of nodeEntries) {
try {
const nodeName = nodeEntry.name.replace('.node.js', '');
const nodeType = `${packageName}.${nodeName}`;
logger.info(`Extracting: ${nodeType}`);
// Extract source info
const sourceInfo = await extractor.extractNodeSource(nodeType);
// Check if this is the If node
if (nodeName === 'If') {
// Look for version in the source code
const versionMatch = sourceInfo.sourceCode.match(/version:\s*(\d+)/);
if (versionMatch) {
ifNodeVersion = versionMatch[1];
logger.info(`📍 Found If node version: ${ifNodeVersion}`);
}
}
// Store in database
await docService.storeNode({
nodeType: nodeType,
name: nodeName,
displayName: nodeName,
description: `${nodeName} node from ${packageName}`,
sourceCode: sourceInfo.sourceCode,
credentialCode: sourceInfo.credentialCode,
packageName: packageName,
version: ifNodeVersion || '1',
hasCredentials: !!sourceInfo.credentialCode,
isTrigger: sourceInfo.sourceCode.includes('trigger: true') || nodeName.toLowerCase().includes('trigger'),
isWebhook: sourceInfo.sourceCode.includes('webhook: true') || nodeName.toLowerCase().includes('webhook'),
});
totalExtracted++;
} catch (error) {
logger.error(`Failed to extract ${nodeEntry.name}: ${error}`);
}
}
} catch (error) {
logger.error(`Failed to scan nodes directory: ${error}`);
}
} catch (error) {
logger.error(`Failed to process package ${packageName}: ${error}`);
}
}
logger.info(`\n✅ Extraction complete!`);
logger.info(`📊 Total nodes extracted: ${totalExtracted}`);
if (ifNodeVersion) {
logger.info(`📍 If node version: ${ifNodeVersion}`);
if (ifNodeVersion === '2' || ifNodeVersion === '2.2') {
logger.info('✅ Successfully extracted latest If node (v2+)!');
} else {
logger.warn(`⚠️ If node version is ${ifNodeVersion}, expected v2 or higher`);
}
}
// Close database
docService.close();
}
async function scanForNodes(dirPath) {
const nodes = [];
async function scan(currentPath) {
try {
const entries = await fs.readdir(currentPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(currentPath, entry.name);
if (entry.isFile() && entry.name.endsWith('.node.js')) {
nodes.push({ name: entry.name, path: fullPath });
} else if (entry.isDirectory() && entry.name !== 'node_modules') {
await scan(fullPath);
}
}
} catch (error) {
logger.debug(`Failed to scan directory ${currentPath}: ${error}`);
}
}
await scan(dirPath);
return nodes;
}
// Run extraction
extractNodesFromDocker().catch(error => {
logger.error('Extraction failed:', error);
process.exit(1);
});

116
scripts/extract-nodes-docker.sh Executable file
View File

@@ -0,0 +1,116 @@
#!/bin/bash
set -e
echo "🐳 n8n Node Extraction via Docker"
echo "================================="
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Function to print colored output
print_status() {
echo -e "${GREEN}[$(date +'%H:%M:%S')]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[$(date +'%H:%M:%S')]${NC} ⚠️ $1"
}
print_error() {
echo -e "${RED}[$(date +'%H:%M:%S')]${NC}$1"
}
# Check if Docker is running
if ! docker info > /dev/null 2>&1; then
print_error "Docker is not running. Please start Docker and try again."
exit 1
fi
print_status "Docker is running ✅"
# Clean up any existing containers
print_status "Cleaning up existing containers..."
docker-compose -f docker-compose.extract.yml down -v 2>/dev/null || true
# Build the project first
print_status "Building the project..."
npm run build
# Start the extraction process
print_status "Starting n8n container to extract latest nodes..."
docker-compose -f docker-compose.extract.yml up -d n8n-latest
# Wait for n8n container to be healthy
print_status "Waiting for n8n container to initialize..."
ATTEMPTS=0
MAX_ATTEMPTS=60
while [ $ATTEMPTS -lt $MAX_ATTEMPTS ]; do
if docker-compose -f docker-compose.extract.yml ps | grep -q "healthy"; then
print_status "n8n container is ready ✅"
break
fi
ATTEMPTS=$((ATTEMPTS + 1))
echo -n "."
sleep 2
done
if [ $ATTEMPTS -eq $MAX_ATTEMPTS ]; then
print_error "n8n container failed to become healthy"
docker-compose -f docker-compose.extract.yml logs n8n-latest
docker-compose -f docker-compose.extract.yml down -v
exit 1
fi
# Run the extraction
print_status "Running node extraction..."
docker-compose -f docker-compose.extract.yml run --rm node-extractor
# Check the results
print_status "Checking extraction results..."
if [ -f "./data/nodes-fresh.db" ]; then
NODE_COUNT=$(sqlite3 ./data/nodes-fresh.db "SELECT COUNT(*) FROM nodes;" 2>/dev/null || echo "0")
IF_VERSION=$(sqlite3 ./data/nodes-fresh.db "SELECT version FROM nodes WHERE name='n8n-nodes-base.If' LIMIT 1;" 2>/dev/null || echo "not found")
print_status "Extracted $NODE_COUNT nodes"
print_status "If node version: $IF_VERSION"
# Check if we got the If node source code and look for version
IF_SOURCE=$(sqlite3 ./data/nodes-fresh.db "SELECT source_code FROM nodes WHERE name='n8n-nodes-base.If' LIMIT 1;" 2>/dev/null || echo "")
if [[ $IF_SOURCE =~ version:[[:space:]]*([0-9]+) ]]; then
IF_CODE_VERSION="${BASH_REMATCH[1]}"
print_status "If node version from source code: v$IF_CODE_VERSION"
if [ "$IF_CODE_VERSION" -ge "2" ]; then
print_status "✅ Successfully extracted latest If node (v$IF_CODE_VERSION)!"
else
print_warning "If node is still v$IF_CODE_VERSION, expected v2 or higher"
fi
fi
else
print_error "Database file not found after extraction"
fi
# Clean up
print_status "Cleaning up Docker containers..."
docker-compose -f docker-compose.extract.yml down -v
print_status "✨ Extraction complete!"
# Offer to restart the MCP server
echo ""
read -p "Would you like to restart the MCP server with the new nodes? (y/n) " -n 1 -r
echo ""
if [[ $REPLY =~ ^[Yy]$ ]]; then
print_status "Restarting MCP server..."
# Kill any existing server process
pkill -f "node.*dist/index.js" || true
# Start the server
npm start &
print_status "MCP server restarted with fresh node database"
fi

108
scripts/extract-nodes-simple.sh Executable file
View File

@@ -0,0 +1,108 @@
#!/bin/bash
set -e
echo "🐳 Simple n8n Node Extraction via Docker"
echo "======================================="
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Function to print colored output
print_status() {
echo -e "${GREEN}[$(date +'%H:%M:%S')]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[$(date +'%H:%M:%S')]${NC} ⚠️ $1"
}
print_error() {
echo -e "${RED}[$(date +'%H:%M:%S')]${NC}$1"
}
# Check if Docker is running
if ! docker info > /dev/null 2>&1; then
print_error "Docker is not running. Please start Docker and try again."
exit 1
fi
print_status "Docker is running ✅"
# Build the project first
print_status "Building the project..."
npm run build
# Create a temporary directory for extraction
TEMP_DIR=$(mktemp -d)
print_status "Created temporary directory: $TEMP_DIR"
# Run Docker container to copy node files
print_status "Running n8n container to extract nodes..."
docker run --rm -d --name n8n-temp n8nio/n8n:latest sleep 300
# Wait a bit for container to start
sleep 5
# Copy n8n modules from container
print_status "Copying n8n modules from container..."
docker cp n8n-temp:/usr/local/lib/node_modules/n8n/node_modules "$TEMP_DIR/node_modules" || {
print_error "Failed to copy node_modules"
docker stop n8n-temp
rm -rf "$TEMP_DIR"
exit 1
}
# Stop the container
docker stop n8n-temp
# Run our extraction script locally
print_status "Running extraction script..."
NODE_ENV=development \
NODE_DB_PATH=./data/nodes-fresh.db \
N8N_MODULES_PATH="$TEMP_DIR/node_modules" \
node scripts/extract-from-docker.js
# Clean up
print_status "Cleaning up temporary files..."
rm -rf "$TEMP_DIR"
# Check the results
print_status "Checking extraction results..."
if [ -f "./data/nodes-fresh.db" ]; then
NODE_COUNT=$(sqlite3 ./data/nodes-fresh.db "SELECT COUNT(*) FROM nodes;" 2>/dev/null || echo "0")
print_status "Extracted $NODE_COUNT nodes"
# Check if we got the If node source code and look for version
IF_SOURCE=$(sqlite3 ./data/nodes-fresh.db "SELECT source_code FROM nodes WHERE node_type='n8n-nodes-base.If' LIMIT 1;" 2>/dev/null || echo "")
if [[ $IF_SOURCE =~ version:[[:space:]]*([0-9]+) ]]; then
IF_CODE_VERSION="${BASH_REMATCH[1]}"
print_status "If node version from source code: v$IF_CODE_VERSION"
if [ "$IF_CODE_VERSION" -ge "2" ]; then
print_status "✅ Successfully extracted latest If node (v$IF_CODE_VERSION)!"
else
print_warning "If node is still v$IF_CODE_VERSION, expected v2 or higher"
fi
fi
else
print_error "Database file not found after extraction"
fi
print_status "✨ Extraction complete!"
# Offer to restart the MCP server
echo ""
read -p "Would you like to restart the MCP server with the new nodes? (y/n) " -n 1 -r
echo ""
if [[ $REPLY =~ ^[Yy]$ ]]; then
print_status "Restarting MCP server..."
# Kill any existing server process
pkill -f "node.*dist/index.js" || true
# Start the server
npm start &
print_status "MCP server restarted with fresh node database"
fi

View File

@@ -0,0 +1,220 @@
#!/usr/bin/env node
import * as dotenv from 'dotenv';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { NodeSourceExtractor } from '../utils/node-source-extractor';
import { logger } from '../utils/logger';
import * as fs from 'fs/promises';
import * as path from 'path';
// Load environment variables
dotenv.config();
async function extractNodesFromDocker() {
logger.info('🐳 Starting Docker-based node extraction...');
// Add Docker volume paths to environment for NodeSourceExtractor
const dockerVolumePaths = [
process.env.N8N_MODULES_PATH || '/n8n-modules',
process.env.N8N_CUSTOM_PATH || '/n8n-custom',
];
logger.info(`Docker volume paths: ${dockerVolumePaths.join(', ')}`);
// Check if volumes are mounted
for (const volumePath of dockerVolumePaths) {
try {
await fs.access(volumePath);
logger.info(`✅ Volume mounted: ${volumePath}`);
// List what's in the volume
const entries = await fs.readdir(volumePath);
logger.info(`Contents of ${volumePath}: ${entries.slice(0, 10).join(', ')}${entries.length > 10 ? '...' : ''}`);
} catch (error) {
logger.warn(`❌ Volume not accessible: ${volumePath}`);
}
}
// Initialize services
const docService = new NodeDocumentationService();
const extractor = new NodeSourceExtractor();
// Extend the extractor's search paths with Docker volumes
(extractor as any).n8nBasePaths.unshift(...dockerVolumePaths);
// Clear existing nodes to ensure we only have latest versions
logger.info('🧹 Clearing existing nodes...');
const db = (docService as any).db;
db.prepare('DELETE FROM nodes').run();
logger.info('🔍 Searching for n8n nodes in Docker volumes...');
// Known n8n packages to extract
const n8nPackages = [
'n8n-nodes-base',
'@n8n/n8n-nodes-langchain',
'n8n-nodes-extras',
];
let totalExtracted = 0;
let ifNodeVersion = null;
for (const packageName of n8nPackages) {
logger.info(`\n📦 Processing package: ${packageName}`);
try {
// Find package in Docker volumes
let packagePath = null;
for (const volumePath of dockerVolumePaths) {
const possiblePaths = [
path.join(volumePath, packageName),
path.join(volumePath, '.pnpm', `${packageName}@*`, 'node_modules', packageName),
];
for (const testPath of possiblePaths) {
try {
// Use glob pattern to find pnpm packages
if (testPath.includes('*')) {
const baseDir = path.dirname(testPath.split('*')[0]);
const entries = await fs.readdir(baseDir);
for (const entry of entries) {
if (entry.includes(packageName.replace('/', '+'))) {
const fullPath = path.join(baseDir, entry, 'node_modules', packageName);
try {
await fs.access(fullPath);
packagePath = fullPath;
break;
} catch {}
}
}
} else {
await fs.access(testPath);
packagePath = testPath;
break;
}
} catch {}
}
if (packagePath) break;
}
if (!packagePath) {
logger.warn(`Package ${packageName} not found in Docker volumes`);
continue;
}
logger.info(`Found package at: ${packagePath}`);
// Check package version
try {
const packageJsonPath = path.join(packagePath, 'package.json');
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf-8'));
logger.info(`Package version: ${packageJson.version}`);
} catch {}
// Find nodes directory
const nodesPath = path.join(packagePath, 'dist', 'nodes');
try {
await fs.access(nodesPath);
logger.info(`Scanning nodes directory: ${nodesPath}`);
// Extract all nodes from this package
const nodeEntries = await scanForNodes(nodesPath);
logger.info(`Found ${nodeEntries.length} nodes in ${packageName}`);
for (const nodeEntry of nodeEntries) {
try {
const nodeName = nodeEntry.name.replace('.node.js', '');
const nodeType = `${packageName}.${nodeName}`;
logger.info(`Extracting: ${nodeType}`);
// Extract source info
const sourceInfo = await extractor.extractNodeSource(nodeType);
// Check if this is the If node
if (nodeName === 'If') {
// Look for version in the source code
const versionMatch = sourceInfo.sourceCode.match(/version:\s*(\d+)/);
if (versionMatch) {
ifNodeVersion = versionMatch[1];
logger.info(`📍 Found If node version: ${ifNodeVersion}`);
}
}
// Store in database
await docService.storeNode({
nodeType: nodeType,
name: nodeName,
displayName: nodeName,
description: `${nodeName} node from ${packageName}`,
sourceCode: sourceInfo.sourceCode,
credentialCode: sourceInfo.credentialCode,
packageName: packageName,
version: ifNodeVersion || '1',
hasCredentials: !!sourceInfo.credentialCode,
isTrigger: sourceInfo.sourceCode.includes('trigger: true') || nodeName.toLowerCase().includes('trigger'),
isWebhook: sourceInfo.sourceCode.includes('webhook: true') || nodeName.toLowerCase().includes('webhook'),
});
totalExtracted++;
} catch (error) {
logger.error(`Failed to extract ${nodeEntry.name}: ${error}`);
}
}
} catch (error) {
logger.error(`Failed to scan nodes directory: ${error}`);
}
} catch (error) {
logger.error(`Failed to process package ${packageName}: ${error}`);
}
}
logger.info(`\n✅ Extraction complete!`);
logger.info(`📊 Total nodes extracted: ${totalExtracted}`);
if (ifNodeVersion) {
logger.info(`📍 If node version: ${ifNodeVersion}`);
if (ifNodeVersion === '2' || ifNodeVersion === '2.2') {
logger.info('✅ Successfully extracted latest If node (v2+)!');
} else {
logger.warn(`⚠️ If node version is ${ifNodeVersion}, expected v2 or higher`);
}
}
// Close database
await docService.close();
}
async function scanForNodes(dirPath: string): Promise<{ name: string; path: string }[]> {
const nodes: { name: string; path: string }[] = [];
async function scan(currentPath: string) {
try {
const entries = await fs.readdir(currentPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(currentPath, entry.name);
if (entry.isFile() && entry.name.endsWith('.node.js')) {
nodes.push({ name: entry.name, path: fullPath });
} else if (entry.isDirectory() && entry.name !== 'node_modules') {
await scan(fullPath);
}
}
} catch (error) {
logger.debug(`Failed to scan directory ${currentPath}: ${error}`);
}
}
await scan(dirPath);
return nodes;
}
// Run extraction
extractNodesFromDocker().catch(error => {
logger.error('Extraction failed:', error);
process.exit(1);
});

View File

@@ -1,8 +1,12 @@
#!/usr/bin/env node
import * as dotenv from 'dotenv';
import { NodeDocumentationService } from '../services/node-documentation-service';
import { logger } from '../utils/logger';
// Load environment variables
dotenv.config();
/**
* Rebuild the enhanced documentation database
*/