feat: Complete overhaul to enhanced documentation-only MCP server
- Removed all workflow execution capabilities per user requirements - Implemented enhanced documentation extraction with operations and API mappings - Fixed credential code extraction for all nodes - Fixed package info extraction (name and version) - Enhanced operations parser to handle n8n markdown format - Fixed documentation search to prioritize app nodes over trigger nodes - Comprehensive test coverage for Slack node extraction - All node information now includes: - Complete operations list (42 for Slack) - API method mappings with documentation URLs - Source code and credential definitions - Package metadata - Related resources and templates 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { NodeDocumentationService } from '../services/node-documentation-service';
|
||||
import { logger } from '../utils/logger';
|
||||
|
||||
async function rebuildDatabase() {
|
||||
console.log('🔄 Starting complete database rebuild...\n');
|
||||
|
||||
const service = new NodeDocumentationService();
|
||||
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
|
||||
console.log('1️⃣ Initializing services...');
|
||||
console.log('2️⃣ Fetching n8n-docs repository...');
|
||||
console.log('3️⃣ Discovering available nodes...');
|
||||
console.log('4️⃣ Extracting node information...\n');
|
||||
|
||||
const stats = await service.rebuildDatabase();
|
||||
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log('\n📊 Rebuild Results:');
|
||||
console.log(` Total nodes processed: ${stats.total}`);
|
||||
console.log(` Successfully stored: ${stats.successful}`);
|
||||
console.log(` Failed: ${stats.failed}`);
|
||||
console.log(` Duration: ${duration}s`);
|
||||
|
||||
if (stats.errors.length > 0) {
|
||||
console.log('\n⚠️ First 5 errors:');
|
||||
stats.errors.slice(0, 5).forEach(error => {
|
||||
console.log(` - ${error}`);
|
||||
});
|
||||
if (stats.errors.length > 5) {
|
||||
console.log(` ... and ${stats.errors.length - 5} more errors`);
|
||||
}
|
||||
}
|
||||
|
||||
// Get final statistics
|
||||
const dbStats = service.getStatistics();
|
||||
console.log('\n📈 Database Statistics:');
|
||||
console.log(` Total nodes: ${dbStats.totalNodes}`);
|
||||
console.log(` Nodes with documentation: ${dbStats.nodesWithDocs}`);
|
||||
console.log(` Nodes with examples: ${dbStats.nodesWithExamples}`);
|
||||
console.log(` Trigger nodes: ${dbStats.triggerNodes}`);
|
||||
console.log(` Webhook nodes: ${dbStats.webhookNodes}`);
|
||||
console.log(` Total packages: ${dbStats.totalPackages}`);
|
||||
|
||||
console.log('\n✨ Database rebuild complete!');
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Database rebuild failed:', error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
service.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
rebuildDatabase().catch(error => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { rebuildDatabase };
|
||||
@@ -1,129 +1,79 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { NodeSourceExtractor } from '../utils/node-source-extractor';
|
||||
import { SQLiteStorageService } from '../services/sqlite-storage-service';
|
||||
import { NodeDocumentationService } from '../services/node-documentation-service';
|
||||
import { logger } from '../utils/logger';
|
||||
import * as path from 'path';
|
||||
|
||||
/**
|
||||
* Rebuild the entire nodes database by extracting all available nodes
|
||||
* Rebuild the enhanced documentation database
|
||||
*/
|
||||
async function rebuildDatabase() {
|
||||
console.log('🔄 Starting database rebuild...\n');
|
||||
async function rebuildDocumentationDatabase() {
|
||||
console.log('🔄 Starting enhanced documentation database rebuild...\n');
|
||||
|
||||
const startTime = Date.now();
|
||||
const extractor = new NodeSourceExtractor();
|
||||
const storage = new SQLiteStorageService();
|
||||
const service = new NodeDocumentationService();
|
||||
|
||||
try {
|
||||
// Step 1: Clear existing database
|
||||
console.log('1️⃣ Clearing existing database...');
|
||||
await storage.rebuildDatabase();
|
||||
// Run the rebuild
|
||||
const results = await service.rebuildDatabase();
|
||||
|
||||
// Step 2: Get all available nodes
|
||||
console.log('2️⃣ Discovering available nodes...');
|
||||
const allNodes = await extractor.listAvailableNodes();
|
||||
console.log(` Found ${allNodes.length} nodes\n`);
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
|
||||
// Step 3: Extract and store each node
|
||||
console.log('3️⃣ Extracting and storing nodes...');
|
||||
let processed = 0;
|
||||
let stored = 0;
|
||||
let failed = 0;
|
||||
const errors: Array<{ node: string; error: string }> = [];
|
||||
console.log('\n✅ Enhanced documentation database rebuild completed!\n');
|
||||
console.log('📊 Results:');
|
||||
console.log(` Total nodes found: ${results.total}`);
|
||||
console.log(` Successfully processed: ${results.successful}`);
|
||||
console.log(` Failed: ${results.failed}`);
|
||||
console.log(` Duration: ${duration}s`);
|
||||
|
||||
// Process in batches for better performance
|
||||
const batchSize = 50;
|
||||
for (let i = 0; i < allNodes.length; i += batchSize) {
|
||||
const batch = allNodes.slice(i, Math.min(i + batchSize, allNodes.length));
|
||||
const nodeInfos = [];
|
||||
if (results.errors.length > 0) {
|
||||
console.log(`\n⚠️ First ${Math.min(5, results.errors.length)} errors:`);
|
||||
results.errors.slice(0, 5).forEach(err => {
|
||||
console.log(` - ${err}`);
|
||||
});
|
||||
|
||||
for (const node of batch) {
|
||||
processed++;
|
||||
|
||||
try {
|
||||
const nodeType = node.packageName ? `${node.packageName}.${node.name}` : node.name;
|
||||
|
||||
// Show progress
|
||||
if (processed % 100 === 0) {
|
||||
const progress = ((processed / allNodes.length) * 100).toFixed(1);
|
||||
console.log(` Progress: ${processed}/${allNodes.length} (${progress}%)`);
|
||||
}
|
||||
|
||||
const nodeInfo = await extractor.extractNodeSource(nodeType);
|
||||
nodeInfos.push(nodeInfo);
|
||||
stored++;
|
||||
} catch (error) {
|
||||
failed++;
|
||||
const errorMsg = error instanceof Error ? error.message : 'Unknown error';
|
||||
errors.push({
|
||||
node: node.name,
|
||||
error: errorMsg
|
||||
});
|
||||
|
||||
// Log first few errors
|
||||
if (errors.length <= 5) {
|
||||
logger.debug(`Failed to extract ${node.name}: ${errorMsg}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bulk store the batch
|
||||
if (nodeInfos.length > 0) {
|
||||
await storage.bulkStoreNodes(nodeInfos);
|
||||
if (results.errors.length > 5) {
|
||||
console.log(` ... and ${results.errors.length - 5} more errors`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Save statistics
|
||||
console.log('\n4️⃣ Saving statistics...');
|
||||
const stats = await storage.getStatistics();
|
||||
await storage.saveExtractionStats(stats);
|
||||
|
||||
// Step 5: Display results
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log('\n✅ Database rebuild completed!\n');
|
||||
console.log('📊 Results:');
|
||||
console.log(` Total nodes found: ${allNodes.length}`);
|
||||
console.log(` Successfully stored: ${stored}`);
|
||||
console.log(` Failed: ${failed}`);
|
||||
console.log(` Duration: ${duration}s`);
|
||||
console.log(` Database size: ${(stats.totalCodeSize / 1024 / 1024).toFixed(2)} MB`);
|
||||
// Get and display statistics
|
||||
const stats = service.getStatistics();
|
||||
console.log('\n📈 Database Statistics:');
|
||||
console.log(` Total nodes: ${stats.totalNodes}`);
|
||||
console.log(` Nodes with documentation: ${stats.nodesWithDocs}`);
|
||||
console.log(` Nodes with examples: ${stats.nodesWithExamples}`);
|
||||
console.log(` Nodes with credentials: ${stats.nodesWithCredentials}`);
|
||||
console.log(` Trigger nodes: ${stats.triggerNodes}`);
|
||||
console.log(` Webhook nodes: ${stats.webhookNodes}`);
|
||||
|
||||
console.log('\n📦 Package distribution:');
|
||||
stats.packageDistribution.slice(0, 10).forEach(pkg => {
|
||||
stats.packageDistribution.slice(0, 10).forEach((pkg: any) => {
|
||||
console.log(` ${pkg.package}: ${pkg.count} nodes`);
|
||||
});
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log(`\n⚠️ First ${Math.min(5, errors.length)} errors:`);
|
||||
errors.slice(0, 5).forEach(err => {
|
||||
console.log(` - ${err.node}: ${err.error}`);
|
||||
});
|
||||
|
||||
if (errors.length > 5) {
|
||||
console.log(` ... and ${errors.length - 5} more errors`);
|
||||
}
|
||||
}
|
||||
|
||||
// Close database connection
|
||||
storage.close();
|
||||
service.close();
|
||||
|
||||
console.log('\n✨ Database is ready for use!');
|
||||
console.log('\n✨ Enhanced documentation database is ready!');
|
||||
console.log('💡 The database now includes:');
|
||||
console.log(' - Complete node source code');
|
||||
console.log(' - Enhanced documentation with operations and API methods');
|
||||
console.log(' - Code examples and templates');
|
||||
console.log(' - Related resources and required scopes');
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Database rebuild failed:', error);
|
||||
storage.close();
|
||||
console.error('\n❌ Documentation database rebuild failed:', error);
|
||||
service.close();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
rebuildDatabase().catch(error => {
|
||||
rebuildDocumentationDatabase().catch(error => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { rebuildDatabase };
|
||||
export { rebuildDocumentationDatabase };
|
||||
Reference in New Issue
Block a user