mirror of
https://github.com/czlonkowski/n8n-mcp.git
synced 2026-02-06 13:33:11 +00:00
chore: add pre-built dist folder for npx usage
This commit is contained in:
thesved
committed by
Romuald Członkowski
Romuald Członkowski
parent
a70d96a373
commit
5057481e70
411
dist/scripts/fetch-templates.js
vendored
Normal file
411
dist/scripts/fetch-templates.js
vendored
Normal file
@@ -0,0 +1,411 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.fetchTemplates = fetchTemplates;
|
||||
const database_adapter_1 = require("../database/database-adapter");
|
||||
const template_service_1 = require("../templates/template-service");
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const zlib = __importStar(require("zlib"));
|
||||
const dotenv = __importStar(require("dotenv"));
|
||||
dotenv.config();
|
||||
function extractNodeConfigs(templateId, templateName, templateViews, workflowCompressed, metadata) {
|
||||
try {
|
||||
const decompressed = zlib.gunzipSync(Buffer.from(workflowCompressed, 'base64'));
|
||||
const workflow = JSON.parse(decompressed.toString('utf-8'));
|
||||
const configs = [];
|
||||
for (const node of workflow.nodes || []) {
|
||||
if (node.type.includes('stickyNote') || !node.parameters) {
|
||||
continue;
|
||||
}
|
||||
configs.push({
|
||||
node_type: node.type,
|
||||
template_id: templateId,
|
||||
template_name: templateName,
|
||||
template_views: templateViews,
|
||||
node_name: node.name,
|
||||
parameters_json: JSON.stringify(node.parameters),
|
||||
credentials_json: node.credentials ? JSON.stringify(node.credentials) : null,
|
||||
has_credentials: node.credentials ? 1 : 0,
|
||||
has_expressions: detectExpressions(node.parameters) ? 1 : 0,
|
||||
complexity: metadata?.complexity || 'medium',
|
||||
use_cases: JSON.stringify(metadata?.use_cases || [])
|
||||
});
|
||||
}
|
||||
return configs;
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Error extracting configs from template ${templateId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
function detectExpressions(params) {
|
||||
if (!params)
|
||||
return false;
|
||||
const json = JSON.stringify(params);
|
||||
return json.includes('={{') || json.includes('$json') || json.includes('$node');
|
||||
}
|
||||
function insertAndRankConfigs(db, configs) {
|
||||
if (configs.length === 0) {
|
||||
console.log('No configs to insert');
|
||||
return;
|
||||
}
|
||||
const templateIds = [...new Set(configs.map(c => c.template_id))];
|
||||
const placeholders = templateIds.map(() => '?').join(',');
|
||||
db.prepare(`DELETE FROM template_node_configs WHERE template_id IN (${placeholders})`).run(...templateIds);
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO template_node_configs (
|
||||
node_type, template_id, template_name, template_views,
|
||||
node_name, parameters_json, credentials_json,
|
||||
has_credentials, has_expressions, complexity, use_cases
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
for (const config of configs) {
|
||||
insertStmt.run(config.node_type, config.template_id, config.template_name, config.template_views, config.node_name, config.parameters_json, config.credentials_json, config.has_credentials, config.has_expressions, config.complexity, config.use_cases);
|
||||
}
|
||||
db.exec(`
|
||||
UPDATE template_node_configs
|
||||
SET rank = (
|
||||
SELECT COUNT(*) + 1
|
||||
FROM template_node_configs AS t2
|
||||
WHERE t2.node_type = template_node_configs.node_type
|
||||
AND t2.template_views > template_node_configs.template_views
|
||||
)
|
||||
`);
|
||||
db.exec(`
|
||||
DELETE FROM template_node_configs
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM template_node_configs
|
||||
WHERE rank <= 10
|
||||
ORDER BY node_type, rank
|
||||
)
|
||||
`);
|
||||
console.log(`✅ Extracted and ranked ${configs.length} node configurations`);
|
||||
}
|
||||
async function extractTemplateConfigs(db, service) {
|
||||
console.log('📦 Extracting node configurations from templates...');
|
||||
const repository = service.repository;
|
||||
const allTemplates = repository.getAllTemplates();
|
||||
const allConfigs = [];
|
||||
let configsExtracted = 0;
|
||||
for (const template of allTemplates) {
|
||||
if (template.workflow_json_compressed) {
|
||||
const metadata = template.metadata_json ? JSON.parse(template.metadata_json) : null;
|
||||
const configs = extractNodeConfigs(template.id, template.name, template.views, template.workflow_json_compressed, metadata);
|
||||
allConfigs.push(...configs);
|
||||
configsExtracted += configs.length;
|
||||
}
|
||||
}
|
||||
if (allConfigs.length > 0) {
|
||||
insertAndRankConfigs(db, allConfigs);
|
||||
const configStats = db.prepare(`
|
||||
SELECT
|
||||
COUNT(DISTINCT node_type) as node_types,
|
||||
COUNT(*) as total_configs,
|
||||
AVG(rank) as avg_rank
|
||||
FROM template_node_configs
|
||||
`).get();
|
||||
console.log(`📊 Node config stats:`);
|
||||
console.log(` - Unique node types: ${configStats.node_types}`);
|
||||
console.log(` - Total configs stored: ${configStats.total_configs}`);
|
||||
console.log(` - Average rank: ${configStats.avg_rank?.toFixed(1) || 'N/A'}`);
|
||||
}
|
||||
else {
|
||||
console.log('⚠️ No node configurations extracted');
|
||||
}
|
||||
}
|
||||
async function fetchTemplates(mode = 'rebuild', generateMetadata = false, metadataOnly = false, extractOnly = false) {
|
||||
if (extractOnly) {
|
||||
console.log('📦 Extract-only mode: Extracting node configurations from existing templates...\n');
|
||||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||||
try {
|
||||
const tableExists = db.prepare(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table' AND name='template_node_configs'
|
||||
`).get();
|
||||
if (!tableExists) {
|
||||
console.log('📋 Creating template_node_configs table...');
|
||||
const migrationPath = path.join(__dirname, '../../src/database/migrations/add-template-node-configs.sql');
|
||||
const migration = fs.readFileSync(migrationPath, 'utf8');
|
||||
db.exec(migration);
|
||||
console.log('✅ Table created successfully\n');
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('❌ Error checking/creating template_node_configs table:', error);
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
const service = new template_service_1.TemplateService(db);
|
||||
await extractTemplateConfigs(db, service);
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (metadataOnly) {
|
||||
console.log('🤖 Metadata-only mode: Generating metadata for existing templates...\n');
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
console.error('❌ OPENAI_API_KEY not set in environment');
|
||||
process.exit(1);
|
||||
}
|
||||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||||
const service = new template_service_1.TemplateService(db);
|
||||
await generateTemplateMetadata(db, service);
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
const modeEmoji = mode === 'rebuild' ? '🔄' : '⬆️';
|
||||
const modeText = mode === 'rebuild' ? 'Rebuilding' : 'Updating';
|
||||
console.log(`${modeEmoji} ${modeText} n8n workflow templates...\n`);
|
||||
if (generateMetadata) {
|
||||
console.log('🤖 Metadata generation enabled (using OpenAI)\n');
|
||||
}
|
||||
const dataDir = './data';
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
const db = await (0, database_adapter_1.createDatabaseAdapter)('./data/nodes.db');
|
||||
if (mode === 'rebuild') {
|
||||
try {
|
||||
db.exec('DROP TABLE IF EXISTS templates');
|
||||
db.exec('DROP TABLE IF EXISTS templates_fts');
|
||||
console.log('🗑️ Dropped existing templates tables (rebuild mode)\n');
|
||||
const schema = fs.readFileSync(path.join(__dirname, '../../src/database/schema.sql'), 'utf8');
|
||||
db.exec(schema);
|
||||
console.log('📋 Applied database schema\n');
|
||||
}
|
||||
catch (error) {
|
||||
console.error('❌ Error setting up database schema:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
else {
|
||||
console.log('📊 Update mode: Keeping existing templates and schema\n');
|
||||
try {
|
||||
const columns = db.prepare("PRAGMA table_info(templates)").all();
|
||||
const hasMetadataColumn = columns.some((col) => col.name === 'metadata_json');
|
||||
if (!hasMetadataColumn) {
|
||||
console.log('📋 Adding metadata columns to existing schema...');
|
||||
db.exec(`
|
||||
ALTER TABLE templates ADD COLUMN metadata_json TEXT;
|
||||
ALTER TABLE templates ADD COLUMN metadata_generated_at DATETIME;
|
||||
`);
|
||||
console.log('✅ Metadata columns added\n');
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.log('📋 Schema is up to date\n');
|
||||
}
|
||||
}
|
||||
const service = new template_service_1.TemplateService(db);
|
||||
let lastMessage = '';
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
await service.fetchAndUpdateTemplates((message, current, total) => {
|
||||
if (lastMessage) {
|
||||
process.stdout.write('\r' + ' '.repeat(lastMessage.length) + '\r');
|
||||
}
|
||||
const progress = total > 0 ? Math.round((current / total) * 100) : 0;
|
||||
lastMessage = `📊 ${message}: ${current}/${total} (${progress}%)`;
|
||||
process.stdout.write(lastMessage);
|
||||
}, mode);
|
||||
console.log('\n');
|
||||
const stats = await service.getTemplateStats();
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
console.log('✅ Template fetch complete!\n');
|
||||
console.log('📈 Statistics:');
|
||||
console.log(` - Total templates: ${stats.totalTemplates}`);
|
||||
console.log(` - Average views: ${stats.averageViews}`);
|
||||
console.log(` - Time elapsed: ${elapsed} seconds`);
|
||||
console.log('\n🔝 Top used nodes:');
|
||||
stats.topUsedNodes.forEach((node, index) => {
|
||||
console.log(` ${index + 1}. ${node.node} (${node.count} templates)`);
|
||||
});
|
||||
console.log('');
|
||||
await extractTemplateConfigs(db, service);
|
||||
if (generateMetadata && process.env.OPENAI_API_KEY) {
|
||||
console.log('\n🤖 Generating metadata for templates...');
|
||||
await generateTemplateMetadata(db, service);
|
||||
}
|
||||
else if (generateMetadata && !process.env.OPENAI_API_KEY) {
|
||||
console.log('\n⚠️ Metadata generation requested but OPENAI_API_KEY not set');
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('\n❌ Error fetching templates:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
if ('close' in db && typeof db.close === 'function') {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
async function generateTemplateMetadata(db, service) {
|
||||
try {
|
||||
const { BatchProcessor } = await Promise.resolve().then(() => __importStar(require('../templates/batch-processor')));
|
||||
const repository = service.repository;
|
||||
const limit = parseInt(process.env.METADATA_LIMIT || '0');
|
||||
const templatesWithoutMetadata = limit > 0
|
||||
? repository.getTemplatesWithoutMetadata(limit)
|
||||
: repository.getTemplatesWithoutMetadata(999999);
|
||||
if (templatesWithoutMetadata.length === 0) {
|
||||
console.log('✅ All templates already have metadata');
|
||||
return;
|
||||
}
|
||||
console.log(`Found ${templatesWithoutMetadata.length} templates without metadata`);
|
||||
const batchSize = parseInt(process.env.OPENAI_BATCH_SIZE || '50');
|
||||
console.log(`Processing in batches of ${batchSize} templates each`);
|
||||
if (batchSize > 100) {
|
||||
console.log(`⚠️ Large batch size (${batchSize}) may take longer to process`);
|
||||
console.log(` Consider using OPENAI_BATCH_SIZE=50 for faster results`);
|
||||
}
|
||||
const processor = new BatchProcessor({
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
model: process.env.OPENAI_MODEL || 'gpt-4o-mini',
|
||||
batchSize: batchSize,
|
||||
outputDir: './temp/batch'
|
||||
});
|
||||
const requests = templatesWithoutMetadata.map((t) => {
|
||||
let workflow = undefined;
|
||||
try {
|
||||
if (t.workflow_json_compressed) {
|
||||
const decompressed = zlib.gunzipSync(Buffer.from(t.workflow_json_compressed, 'base64'));
|
||||
workflow = JSON.parse(decompressed.toString());
|
||||
}
|
||||
else if (t.workflow_json) {
|
||||
workflow = JSON.parse(t.workflow_json);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.warn(`Failed to parse workflow for template ${t.id}:`, error);
|
||||
}
|
||||
let nodes = [];
|
||||
try {
|
||||
if (t.nodes_used) {
|
||||
nodes = JSON.parse(t.nodes_used);
|
||||
if (!Array.isArray(nodes)) {
|
||||
console.warn(`Template ${t.id} has invalid nodes_used (not an array), using empty array`);
|
||||
nodes = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.warn(`Failed to parse nodes_used for template ${t.id}:`, error);
|
||||
nodes = [];
|
||||
}
|
||||
return {
|
||||
templateId: t.id,
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
nodes: nodes,
|
||||
workflow
|
||||
};
|
||||
});
|
||||
const results = await processor.processTemplates(requests, (message, current, total) => {
|
||||
process.stdout.write(`\r📊 ${message}: ${current}/${total}`);
|
||||
});
|
||||
console.log('\n');
|
||||
const metadataMap = new Map();
|
||||
for (const [templateId, result] of results) {
|
||||
if (!result.error) {
|
||||
metadataMap.set(templateId, result.metadata);
|
||||
}
|
||||
}
|
||||
if (metadataMap.size > 0) {
|
||||
repository.batchUpdateMetadata(metadataMap);
|
||||
console.log(`✅ Updated metadata for ${metadataMap.size} templates`);
|
||||
}
|
||||
const stats = repository.getMetadataStats();
|
||||
console.log('\n📈 Metadata Statistics:');
|
||||
console.log(` - Total templates: ${stats.total}`);
|
||||
console.log(` - With metadata: ${stats.withMetadata}`);
|
||||
console.log(` - Without metadata: ${stats.withoutMetadata}`);
|
||||
console.log(` - Outdated (>30 days): ${stats.outdated}`);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('\n❌ Error generating metadata:', error);
|
||||
}
|
||||
}
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let mode = 'rebuild';
|
||||
let generateMetadata = false;
|
||||
let metadataOnly = false;
|
||||
let extractOnly = false;
|
||||
const modeIndex = args.findIndex(arg => arg.startsWith('--mode'));
|
||||
if (modeIndex !== -1) {
|
||||
const modeArg = args[modeIndex];
|
||||
const modeValue = modeArg.includes('=') ? modeArg.split('=')[1] : args[modeIndex + 1];
|
||||
if (modeValue === 'update') {
|
||||
mode = 'update';
|
||||
}
|
||||
}
|
||||
if (args.includes('--update')) {
|
||||
mode = 'update';
|
||||
}
|
||||
if (args.includes('--generate-metadata') || args.includes('--metadata')) {
|
||||
generateMetadata = true;
|
||||
}
|
||||
if (args.includes('--metadata-only')) {
|
||||
metadataOnly = true;
|
||||
}
|
||||
if (args.includes('--extract-only') || args.includes('--extract')) {
|
||||
extractOnly = true;
|
||||
}
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log('Usage: npm run fetch:templates [options]\n');
|
||||
console.log('Options:');
|
||||
console.log(' --mode=rebuild|update Rebuild from scratch or update existing (default: rebuild)');
|
||||
console.log(' --update Shorthand for --mode=update');
|
||||
console.log(' --generate-metadata Generate AI metadata after fetching templates');
|
||||
console.log(' --metadata Shorthand for --generate-metadata');
|
||||
console.log(' --metadata-only Only generate metadata, skip template fetching');
|
||||
console.log(' --extract-only Only extract node configs, skip template fetching');
|
||||
console.log(' --extract Shorthand for --extract-only');
|
||||
console.log(' --help, -h Show this help message');
|
||||
process.exit(0);
|
||||
}
|
||||
return { mode, generateMetadata, metadataOnly, extractOnly };
|
||||
}
|
||||
if (require.main === module) {
|
||||
const { mode, generateMetadata, metadataOnly, extractOnly } = parseArgs();
|
||||
fetchTemplates(mode, generateMetadata, metadataOnly, extractOnly).catch(console.error);
|
||||
}
|
||||
//# sourceMappingURL=fetch-templates.js.map
|
||||
Reference in New Issue
Block a user