mirror of
https://github.com/czlonkowski/n8n-mcp.git
synced 2026-01-30 06:22:04 +00:00
411 lines
18 KiB
JavaScript
411 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
||
"use strict";
|
||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||
if (k2 === undefined) k2 = k;
|
||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||
}
|
||
Object.defineProperty(o, k2, desc);
|
||
}) : (function(o, m, k, k2) {
|
||
if (k2 === undefined) k2 = k;
|
||
o[k2] = m[k];
|
||
}));
|
||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||
}) : function(o, v) {
|
||
o["default"] = v;
|
||
});
|
||
var __importStar = (this && this.__importStar) || (function () {
|
||
var ownKeys = function(o) {
|
||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||
var ar = [];
|
||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||
return ar;
|
||
};
|
||
return ownKeys(o);
|
||
};
|
||
return function (mod) {
|
||
if (mod && mod.__esModule) return mod;
|
||
var result = {};
|
||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||
__setModuleDefault(result, mod);
|
||
return result;
|
||
};
|
||
})();
|
||
Object.defineProperty(exports, "__esModule", { value: true });
|
||
exports.fetchTemplates = fetchTemplates;
|
||
const database_adapter_1 = require("../database/database-adapter");
|
||
const template_service_1 = require("../templates/template-service");
|
||
const fs = __importStar(require("fs"));
|
||
const path = __importStar(require("path"));
|
||
const zlib = __importStar(require("zlib"));
|
||
const dotenv = __importStar(require("dotenv"));
|
||
dotenv.config();
|
||
function extractNodeConfigs(templateId, templateName, templateViews, workflowCompressed, metadata) {
|
||
try {
|
||
const decompressed = zlib.gunzipSync(Buffer.from(workflowCompressed, 'base64'));
|
||
const workflow = JSON.parse(decompressed.toString('utf-8'));
|
||
const configs = [];
|
||
for (const node of workflow.nodes || []) {
|
||
if (node.type.includes('stickyNote') || !node.parameters) {
|
||
continue;
|
||
}
|
||
configs.push({
|
||
node_type: node.type,
|
||
template_id: templateId,
|
||
template_name: templateName,
|
||
template_views: templateViews,
|
||
node_name: node.name,
|
||
parameters_json: JSON.stringify(node.parameters),
|
||
credentials_json: node.credentials ? JSON.stringify(node.credentials) : null,
|
||
has_credentials: node.credentials ? 1 : 0,
|
||
has_expressions: detectExpressions(node.parameters) ? 1 : 0,
|
||
complexity: metadata?.complexity || 'medium',
|
||
use_cases: JSON.stringify(metadata?.use_cases || [])
|
||
});
|
||
}
|
||
return configs;
|
||
}
|
||
catch (error) {
|
||
console.error(`Error extracting configs from template ${templateId}:`, error);
|
||
return [];
|
||
}
|
||
}
|
||
function detectExpressions(params) {
|
||
if (!params)
|
||
return false;
|
||
const json = JSON.stringify(params);
|
||
return json.includes('={{') || json.includes('$json') || json.includes('$node');
|
||
}
|
||
function insertAndRankConfigs(db, configs) {
|
||
if (configs.length === 0) {
|
||
console.log('No configs to insert');
|
||
return;
|
||
}
|
||
const templateIds = [...new Set(configs.map(c => c.template_id))];
|
||
const placeholders = templateIds.map(() => '?').join(',');
|
||
db.prepare(`DELETE FROM template_node_configs WHERE template_id IN (${placeholders})`).run(...templateIds);
|
||
const insertStmt = db.prepare(`
|
||
INSERT INTO template_node_configs (
|
||
node_type, template_id, template_name, template_views,
|
||
node_name, parameters_json, credentials_json,
|
||
has_credentials, has_expressions, complexity, use_cases
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
`);
|
||
for (const config of configs) {
|
||
insertStmt.run(config.node_type, config.template_id, config.template_name, config.template_views, config.node_name, config.parameters_json, config.credentials_json, config.has_credentials, config.has_expressions, config.complexity, config.use_cases);
|
||
}
|
||
db.exec(`
|
||
UPDATE template_node_configs
|
||
SET rank = (
|
||
SELECT COUNT(*) + 1
|
||
FROM template_node_configs AS t2
|
||
WHERE t2.node_type = template_node_configs.node_type
|
||
AND t2.template_views > template_node_configs.template_views
|
||
)
|
||
`);
|
||
db.exec(`
|
||
DELETE FROM template_node_configs
|
||
WHERE id NOT IN (
|
||
SELECT id FROM template_node_configs
|
||
WHERE rank <= 10
|
||
ORDER BY node_type, rank
|
||
)
|
||
`);
|
||
console.log(`✅ Extracted and ranked ${configs.length} node configurations`);
|
||
}
|
||
async function extractTemplateConfigs(db, service) {
|
||
console.log('📦 Extracting node configurations from templates...');
|
||
const repository = service.repository;
|
||
const allTemplates = repository.getAllTemplates();
|
||
const allConfigs = [];
|
||
let configsExtracted = 0;
|
||
for (const template of allTemplates) {
|
||
if (template.workflow_json_compressed) {
|
||
const metadata = template.metadata_json ? JSON.parse(template.metadata_json) : null;
|
||
const configs = extractNodeConfigs(template.id, template.name, template.views, template.workflow_json_compressed, metadata);
|
||
allConfigs.push(...configs);
|
||
configsExtracted += configs.length;
|
||
}
|
||
}
|
||
if (allConfigs.length > 0) {
|
||
insertAndRankConfigs(db, allConfigs);
|
||
const configStats = db.prepare(`
|
||
SELECT
|
||
COUNT(DISTINCT node_type) as node_types,
|
||
COUNT(*) as total_configs,
|
||
AVG(rank) as avg_rank
|
||
FROM template_node_configs
|
||
`).get();
|
||
console.log(`📊 Node config stats:`);
|
||
console.log(` - Unique node types: ${configStats.node_types}`);
|
||
console.log(` - Total configs stored: ${configStats.total_configs}`);
|
||
console.log(` - Average rank: ${configStats.avg_rank?.toFixed(1) || 'N/A'}`);
|
||
}
|
||
else {
|
||
console.log('⚠️ No node configurations extracted');
|
||
}
|
||
}
|
||
async function fetchTemplates(mode = 'rebuild', generateMetadata = false, metadataOnly = false, extractOnly = false) {
|
||
if (extractOnly) {
|
||
console.log('📦 Extract-only mode: Extracting node configurations from existing templates...\n');
|
||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||
try {
|
||
const tableExists = db.prepare(`
|
||
SELECT name FROM sqlite_master
|
||
WHERE type='table' AND name='template_node_configs'
|
||
`).get();
|
||
if (!tableExists) {
|
||
console.log('📋 Creating template_node_configs table...');
|
||
const migrationPath = path.join(__dirname, '../../src/database/migrations/add-template-node-configs.sql');
|
||
const migration = fs.readFileSync(migrationPath, 'utf8');
|
||
db.exec(migration);
|
||
console.log('✅ Table created successfully\n');
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.error('❌ Error checking/creating template_node_configs table:', error);
|
||
if ('close' in db && typeof db.close === 'function') {
|
||
db.close();
|
||
}
|
||
process.exit(1);
|
||
}
|
||
const service = new template_service_1.TemplateService(db);
|
||
await extractTemplateConfigs(db, service);
|
||
if ('close' in db && typeof db.close === 'function') {
|
||
db.close();
|
||
}
|
||
return;
|
||
}
|
||
if (metadataOnly) {
|
||
console.log('🤖 Metadata-only mode: Generating metadata for existing templates...\n');
|
||
if (!process.env.OPENAI_API_KEY) {
|
||
console.error('❌ OPENAI_API_KEY not set in environment');
|
||
process.exit(1);
|
||
}
|
||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||
const service = new template_service_1.TemplateService(db);
|
||
await generateTemplateMetadata(db, service);
|
||
if ('close' in db && typeof db.close === 'function') {
|
||
db.close();
|
||
}
|
||
return;
|
||
}
|
||
const modeEmoji = mode === 'rebuild' ? '🔄' : '⬆️';
|
||
const modeText = mode === 'rebuild' ? 'Rebuilding' : 'Updating';
|
||
console.log(`${modeEmoji} ${modeText} n8n workflow templates...\n`);
|
||
if (generateMetadata) {
|
||
console.log('🤖 Metadata generation enabled (using OpenAI)\n');
|
||
}
|
||
const dataDir = './data';
|
||
if (!fs.existsSync(dataDir)) {
|
||
fs.mkdirSync(dataDir, { recursive: true });
|
||
}
|
||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||
if (mode === 'rebuild') {
|
||
try {
|
||
db.exec('DROP TABLE IF EXISTS templates');
|
||
db.exec('DROP TABLE IF EXISTS templates_fts');
|
||
console.log('🗑️ Dropped existing templates tables (rebuild mode)\n');
|
||
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
|
||
db.exec(schema);
|
||
console.log('📋 Applied database schema\n');
|
||
}
|
||
catch (error) {
|
||
console.error('❌ Error setting up database schema:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
else {
|
||
console.log('📊 Update mode: Keeping existing templates and schema\n');
|
||
try {
|
||
const columns = db.prepare("PRAGMA table_info(templates)").all();
|
||
const hasMetadataColumn = columns.some((col) => col.name === 'metadata_json');
|
||
if (!hasMetadataColumn) {
|
||
console.log('📋 Adding metadata columns to existing schema...');
|
||
db.exec(`
|
||
ALTER TABLE templates ADD COLUMN metadata_json TEXT;
|
||
ALTER TABLE templates ADD COLUMN metadata_generated_at DATETIME;
|
||
`);
|
||
console.log('✅ Metadata columns added\n');
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.log('📋 Schema is up to date\n');
|
||
}
|
||
}
|
||
const service = new template_service_1.TemplateService(db);
|
||
let lastMessage = '';
|
||
const startTime = Date.now();
|
||
try {
|
||
await service.fetchAndUpdateTemplates((message, current, total) => {
|
||
if (lastMessage) {
|
||
process.stdout.write('\r' + ' '.repeat(lastMessage.length) + '\r');
|
||
}
|
||
const progress = total > 0 ? Math.round((current / total) * 100) : 0;
|
||
lastMessage = `📊 ${message}: ${current}/${total} (${progress}%)`;
|
||
process.stdout.write(lastMessage);
|
||
}, mode);
|
||
console.log('\n');
|
||
const stats = await service.getTemplateStats();
|
||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||
console.log('✅ Template fetch complete!\n');
|
||
console.log('📈 Statistics:');
|
||
console.log(` - Total templates: ${stats.totalTemplates}`);
|
||
console.log(` - Average views: ${stats.averageViews}`);
|
||
console.log(` - Time elapsed: ${elapsed} seconds`);
|
||
console.log('\n🔝 Top used nodes:');
|
||
stats.topUsedNodes.forEach((node, index) => {
|
||
console.log(` ${index + 1}. ${node.node} (${node.count} templates)`);
|
||
});
|
||
console.log('');
|
||
await extractTemplateConfigs(db, service);
|
||
if (generateMetadata && process.env.OPENAI_API_KEY) {
|
||
console.log('\n🤖 Generating metadata for templates...');
|
||
await generateTemplateMetadata(db, service);
|
||
}
|
||
else if (generateMetadata && !process.env.OPENAI_API_KEY) {
|
||
console.log('\n⚠️ Metadata generation requested but OPENAI_API_KEY not set');
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.error('\n❌ Error fetching templates:', error);
|
||
process.exit(1);
|
||
}
|
||
if ('close' in db && typeof db.close === 'function') {
|
||
db.close();
|
||
}
|
||
}
|
||
async function generateTemplateMetadata(db, service) {
|
||
try {
|
||
const { BatchProcessor } = await Promise.resolve().then(() => __importStar(require('../templates/batch-processor')));
|
||
const repository = service.repository;
|
||
const limit = parseInt(process.env.METADATA_LIMIT || '0');
|
||
const templatesWithoutMetadata = limit > 0
|
||
? repository.getTemplatesWithoutMetadata(limit)
|
||
: repository.getTemplatesWithoutMetadata(999999);
|
||
if (templatesWithoutMetadata.length === 0) {
|
||
console.log('✅ All templates already have metadata');
|
||
return;
|
||
}
|
||
console.log(`Found ${templatesWithoutMetadata.length} templates without metadata`);
|
||
const batchSize = parseInt(process.env.OPENAI_BATCH_SIZE || '50');
|
||
console.log(`Processing in batches of ${batchSize} templates each`);
|
||
if (batchSize > 100) {
|
||
console.log(`⚠️ Large batch size (${batchSize}) may take longer to process`);
|
||
console.log(` Consider using OPENAI_BATCH_SIZE=50 for faster results`);
|
||
}
|
||
const processor = new BatchProcessor({
|
||
apiKey: process.env.OPENAI_API_KEY,
|
||
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
|
||
batchSize: batchSize,
|
||
outputDir: './temp/batch'
|
||
});
|
||
const requests = templatesWithoutMetadata.map((t) => {
|
||
let workflow = undefined;
|
||
try {
|
||
if (t.workflow_json_compressed) {
|
||
const decompressed = zlib.gunzipSync(Buffer.from(t.workflow_json_compressed, 'base64'));
|
||
workflow = JSON.parse(decompressed.toString());
|
||
}
|
||
else if (t.workflow_json) {
|
||
workflow = JSON.parse(t.workflow_json);
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.warn(`Failed to parse workflow for template ${t.id}:`, error);
|
||
}
|
||
let nodes = [];
|
||
try {
|
||
if (t.nodes_used) {
|
||
nodes = JSON.parse(t.nodes_used);
|
||
if (!Array.isArray(nodes)) {
|
||
console.warn(`Template ${t.id} has invalid nodes_used (not an array), using empty array`);
|
||
nodes = [];
|
||
}
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.warn(`Failed to parse nodes_used for template ${t.id}:`, error);
|
||
nodes = [];
|
||
}
|
||
return {
|
||
templateId: t.id,
|
||
name: t.name,
|
||
description: t.description,
|
||
nodes: nodes,
|
||
workflow
|
||
};
|
||
});
|
||
const results = await processor.processTemplates(requests, (message, current, total) => {
|
||
process.stdout.write(`\r📊 ${message}: ${current}/${total}`);
|
||
});
|
||
console.log('\n');
|
||
const metadataMap = new Map();
|
||
for (const [templateId, result] of results) {
|
||
if (!result.error) {
|
||
metadataMap.set(templateId, result.metadata);
|
||
}
|
||
}
|
||
if (metadataMap.size > 0) {
|
||
repository.batchUpdateMetadata(metadataMap);
|
||
console.log(`✅ Updated metadata for ${metadataMap.size} templates`);
|
||
}
|
||
const stats = repository.getMetadataStats();
|
||
console.log('\n📈 Metadata Statistics:');
|
||
console.log(` - Total templates: ${stats.total}`);
|
||
console.log(` - With metadata: ${stats.withMetadata}`);
|
||
console.log(` - Without metadata: ${stats.withoutMetadata}`);
|
||
console.log(` - Outdated (>30 days): ${stats.outdated}`);
|
||
}
|
||
catch (error) {
|
||
console.error('\n❌ Error generating metadata:', error);
|
||
}
|
||
}
|
||
function parseArgs() {
|
||
const args = process.argv.slice(2);
|
||
let mode = 'rebuild';
|
||
let generateMetadata = false;
|
||
let metadataOnly = false;
|
||
let extractOnly = false;
|
||
const modeIndex = args.findIndex(arg => arg.startsWith('--mode'));
|
||
if (modeIndex !== -1) {
|
||
const modeArg = args[modeIndex];
|
||
const modeValue = modeArg.includes('=') ? modeArg.split('=')[1] : args[modeIndex + 1];
|
||
if (modeValue === 'update') {
|
||
mode = 'update';
|
||
}
|
||
}
|
||
if (args.includes('--update')) {
|
||
mode = 'update';
|
||
}
|
||
if (args.includes('--generate-metadata') || args.includes('--metadata')) {
|
||
generateMetadata = true;
|
||
}
|
||
if (args.includes('--metadata-only')) {
|
||
metadataOnly = true;
|
||
}
|
||
if (args.includes('--extract-only') || args.includes('--extract')) {
|
||
extractOnly = true;
|
||
}
|
||
if (args.includes('--help') || args.includes('-h')) {
|
||
console.log('Usage: npm run fetch:templates [options]\n');
|
||
console.log('Options:');
|
||
console.log(' --mode=rebuild|update Rebuild from scratch or update existing (default: rebuild)');
|
||
console.log(' --update Shorthand for --mode=update');
|
||
console.log(' --generate-metadata Generate AI metadata after fetching templates');
|
||
console.log(' --metadata Shorthand for --generate-metadata');
|
||
console.log(' --metadata-only Only generate metadata, skip template fetching');
|
||
console.log(' --extract-only Only extract node configs, skip template fetching');
|
||
console.log(' --extract Shorthand for --extract-only');
|
||
console.log(' --help, -h Show this help message');
|
||
process.exit(0);
|
||
}
|
||
return { mode, generateMetadata, metadataOnly, extractOnly };
|
||
}
|
||
if (require.main === module) {
|
||
const { mode, generateMetadata, metadataOnly, extractOnly } = parseArgs();
|
||
fetchTemplates(mode, generateMetadata, metadataOnly, extractOnly).catch(console.error);
|
||
}
|
||
//# sourceMappingURL=fetch-templates.js.map
|