feat: Complete overhaul to enhanced documentation-only MCP server
- Removed all workflow execution capabilities per user requirements - Implemented enhanced documentation extraction with operations and API mappings - Fixed credential code extraction for all nodes - Fixed package info extraction (name and version) - Enhanced operations parser to handle n8n markdown format - Fixed documentation search to prioritize app nodes over trigger nodes - Comprehensive test coverage for Slack node extraction - All node information now includes: - Complete operations list (42 for Slack) - API method mappings with documentation URLs - Source code and credential definitions - Package metadata - Related resources and templates 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
@@ -4,7 +4,15 @@ import path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
import { logger } from '../utils/logger';
|
||||
import { NodeSourceExtractor } from '../utils/node-source-extractor';
|
||||
import { DocumentationFetcher } from '../utils/documentation-fetcher';
|
||||
import {
|
||||
EnhancedDocumentationFetcher,
|
||||
EnhancedNodeDocumentation,
|
||||
OperationInfo,
|
||||
ApiMethodMapping,
|
||||
CodeExample,
|
||||
TemplateInfo,
|
||||
RelatedResource
|
||||
} from '../utils/enhanced-documentation-fetcher';
|
||||
import { ExampleGenerator } from '../utils/example-generator';
|
||||
|
||||
interface NodeInfo {
|
||||
@@ -17,8 +25,15 @@ interface NodeInfo {
|
||||
icon?: string;
|
||||
sourceCode: string;
|
||||
credentialCode?: string;
|
||||
documentation?: string;
|
||||
documentationMarkdown?: string;
|
||||
documentationUrl?: string;
|
||||
documentationTitle?: string;
|
||||
operations?: OperationInfo[];
|
||||
apiMethods?: ApiMethodMapping[];
|
||||
documentationExamples?: CodeExample[];
|
||||
templates?: TemplateInfo[];
|
||||
relatedResources?: RelatedResource[];
|
||||
requiredScopes?: string[];
|
||||
exampleWorkflow?: any;
|
||||
exampleParameters?: any;
|
||||
propertiesSchema?: any;
|
||||
@@ -44,7 +59,7 @@ interface SearchOptions {
|
||||
export class NodeDocumentationService {
|
||||
private db: Database.Database;
|
||||
private extractor: NodeSourceExtractor;
|
||||
private docsFetcher: DocumentationFetcher;
|
||||
private docsFetcher: EnhancedDocumentationFetcher;
|
||||
|
||||
constructor(dbPath?: string) {
|
||||
const databasePath = dbPath || process.env.NODE_DB_PATH || path.join(process.cwd(), 'data', 'nodes-v2.db');
|
||||
@@ -57,7 +72,7 @@ export class NodeDocumentationService {
|
||||
|
||||
this.db = new Database(databasePath);
|
||||
this.extractor = new NodeSourceExtractor();
|
||||
this.docsFetcher = new DocumentationFetcher();
|
||||
this.docsFetcher = new EnhancedDocumentationFetcher();
|
||||
|
||||
// Initialize database with new schema
|
||||
this.initializeDatabase();
|
||||
@@ -88,6 +103,15 @@ CREATE TABLE IF NOT EXISTS nodes (
|
||||
-- Documentation
|
||||
documentation_markdown TEXT,
|
||||
documentation_url TEXT,
|
||||
documentation_title TEXT,
|
||||
|
||||
-- Enhanced documentation fields (stored as JSON)
|
||||
operations TEXT,
|
||||
api_methods TEXT,
|
||||
documentation_examples TEXT,
|
||||
templates TEXT,
|
||||
related_resources TEXT,
|
||||
required_scopes TEXT,
|
||||
|
||||
-- Example usage
|
||||
example_workflow TEXT,
|
||||
@@ -182,14 +206,16 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
node_type, name, display_name, description, category, subcategory, icon,
|
||||
source_code, credential_code, code_hash, code_length,
|
||||
documentation_markdown, documentation_url,
|
||||
documentation_markdown, documentation_url, documentation_title,
|
||||
operations, api_methods, documentation_examples, templates, related_resources, required_scopes,
|
||||
example_workflow, example_parameters, properties_schema,
|
||||
package_name, version, codex_data, aliases,
|
||||
has_credentials, is_trigger, is_webhook
|
||||
) VALUES (
|
||||
@nodeType, @name, @displayName, @description, @category, @subcategory, @icon,
|
||||
@sourceCode, @credentialCode, @hash, @codeLength,
|
||||
@documentation, @documentationUrl,
|
||||
@documentation, @documentationUrl, @documentationTitle,
|
||||
@operations, @apiMethods, @documentationExamples, @templates, @relatedResources, @requiredScopes,
|
||||
@exampleWorkflow, @exampleParameters, @propertiesSchema,
|
||||
@packageName, @version, @codexData, @aliases,
|
||||
@hasCredentials, @isTrigger, @isWebhook
|
||||
@@ -208,8 +234,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
credentialCode: nodeInfo.credentialCode || null,
|
||||
hash,
|
||||
codeLength: nodeInfo.sourceCode.length,
|
||||
documentation: nodeInfo.documentation || null,
|
||||
documentation: nodeInfo.documentationMarkdown || null,
|
||||
documentationUrl: nodeInfo.documentationUrl || null,
|
||||
documentationTitle: nodeInfo.documentationTitle || null,
|
||||
operations: nodeInfo.operations ? JSON.stringify(nodeInfo.operations) : null,
|
||||
apiMethods: nodeInfo.apiMethods ? JSON.stringify(nodeInfo.apiMethods) : null,
|
||||
documentationExamples: nodeInfo.documentationExamples ? JSON.stringify(nodeInfo.documentationExamples) : null,
|
||||
templates: nodeInfo.templates ? JSON.stringify(nodeInfo.templates) : null,
|
||||
relatedResources: nodeInfo.relatedResources ? JSON.stringify(nodeInfo.relatedResources) : null,
|
||||
requiredScopes: nodeInfo.requiredScopes ? JSON.stringify(nodeInfo.requiredScopes) : null,
|
||||
exampleWorkflow: nodeInfo.exampleWorkflow ? JSON.stringify(nodeInfo.exampleWorkflow) : null,
|
||||
exampleParameters: nodeInfo.exampleParameters ? JSON.stringify(nodeInfo.exampleParameters) : null,
|
||||
propertiesSchema: nodeInfo.propertiesSchema ? JSON.stringify(nodeInfo.propertiesSchema) : null,
|
||||
@@ -346,13 +379,13 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
// Parse node definition to get metadata
|
||||
const nodeDefinition = this.parseNodeDefinition(nodeData.sourceCode);
|
||||
|
||||
// Get documentation
|
||||
const docs = await this.docsFetcher.getNodeDocumentation(nodeType);
|
||||
// Get enhanced documentation
|
||||
const enhancedDocs = await this.docsFetcher.getEnhancedNodeDocumentation(nodeType);
|
||||
|
||||
// Generate example
|
||||
const example = ExampleGenerator.generateFromNodeDefinition(nodeDefinition);
|
||||
|
||||
// Prepare node info
|
||||
// Prepare node info with enhanced documentation
|
||||
const nodeInfo: NodeInfo = {
|
||||
nodeType: nodeType,
|
||||
name: node.name,
|
||||
@@ -363,8 +396,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
icon: nodeDefinition.icon,
|
||||
sourceCode: nodeData.sourceCode,
|
||||
credentialCode: nodeData.credentialCode,
|
||||
documentation: docs?.markdown,
|
||||
documentationUrl: docs?.url,
|
||||
documentationMarkdown: enhancedDocs?.markdown,
|
||||
documentationUrl: enhancedDocs?.url,
|
||||
documentationTitle: enhancedDocs?.title,
|
||||
operations: enhancedDocs?.operations,
|
||||
apiMethods: enhancedDocs?.apiMethods,
|
||||
documentationExamples: enhancedDocs?.examples,
|
||||
templates: enhancedDocs?.templates,
|
||||
relatedResources: enhancedDocs?.relatedResources,
|
||||
requiredScopes: enhancedDocs?.requiredScopes,
|
||||
exampleWorkflow: example,
|
||||
exampleParameters: example.nodes[0]?.parameters,
|
||||
propertiesSchema: nodeDefinition.properties,
|
||||
@@ -410,28 +450,88 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
* Parse node definition from source code
|
||||
*/
|
||||
private parseNodeDefinition(sourceCode: string): any {
|
||||
try {
|
||||
// Try to extract the description object from the source
|
||||
const descMatch = sourceCode.match(/description\s*[:=]\s*({[\s\S]*?})\s*[,;]/);
|
||||
if (descMatch) {
|
||||
// Clean up the match and try to parse it
|
||||
const descStr = descMatch[1]
|
||||
.replace(/(['"])?([a-zA-Z0-9_]+)(['"])?\s*:/g, '"$2":') // Quote property names
|
||||
.replace(/:\s*'([^']*)'/g, ': "$1"') // Convert single quotes to double
|
||||
.replace(/,\s*}/g, '}'); // Remove trailing commas
|
||||
|
||||
return JSON.parse(descStr);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug('Failed to parse node definition:', error);
|
||||
}
|
||||
|
||||
// Return minimal definition if parsing fails
|
||||
return {
|
||||
const result: any = {
|
||||
displayName: '',
|
||||
description: '',
|
||||
properties: []
|
||||
properties: [],
|
||||
category: null,
|
||||
subcategory: null,
|
||||
icon: null,
|
||||
version: null,
|
||||
codex: null,
|
||||
alias: null
|
||||
};
|
||||
|
||||
try {
|
||||
// Extract individual properties using specific patterns
|
||||
|
||||
// Display name
|
||||
const displayNameMatch = sourceCode.match(/displayName\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
||||
if (displayNameMatch) {
|
||||
result.displayName = displayNameMatch[1];
|
||||
}
|
||||
|
||||
// Description
|
||||
const descriptionMatch = sourceCode.match(/description\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
||||
if (descriptionMatch) {
|
||||
result.description = descriptionMatch[1];
|
||||
}
|
||||
|
||||
// Icon
|
||||
const iconMatch = sourceCode.match(/icon\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
||||
if (iconMatch) {
|
||||
result.icon = iconMatch[1];
|
||||
}
|
||||
|
||||
// Category/group
|
||||
const groupMatch = sourceCode.match(/group\s*[:=]\s*\[['"`]([^'"`]+)['"`]\]/);
|
||||
if (groupMatch) {
|
||||
result.category = groupMatch[1];
|
||||
}
|
||||
|
||||
// Version
|
||||
const versionMatch = sourceCode.match(/version\s*[:=]\s*(\d+)/);
|
||||
if (versionMatch) {
|
||||
result.version = parseInt(versionMatch[1]);
|
||||
}
|
||||
|
||||
// Subtitle
|
||||
const subtitleMatch = sourceCode.match(/subtitle\s*[:=]\s*['"`]([^'"`]+)['"`]/);
|
||||
if (subtitleMatch) {
|
||||
result.subtitle = subtitleMatch[1];
|
||||
}
|
||||
|
||||
// Try to extract properties array
|
||||
const propsMatch = sourceCode.match(/properties\s*[:=]\s*(\[[\s\S]*?\])\s*[,}]/);
|
||||
if (propsMatch) {
|
||||
try {
|
||||
// This is complex to parse from minified code, so we'll skip for now
|
||||
result.properties = [];
|
||||
} catch (e) {
|
||||
// Ignore parsing errors
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it's a trigger node
|
||||
if (sourceCode.includes('implements.*ITrigger') ||
|
||||
sourceCode.includes('polling:.*true') ||
|
||||
sourceCode.includes('webhook:.*true') ||
|
||||
result.displayName.toLowerCase().includes('trigger')) {
|
||||
result.isTrigger = true;
|
||||
}
|
||||
|
||||
// Check if it's a webhook node
|
||||
if (sourceCode.includes('webhooks:') ||
|
||||
sourceCode.includes('webhook:.*true') ||
|
||||
result.displayName.toLowerCase().includes('webhook')) {
|
||||
result.isWebhook = true;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.debug('Error parsing node definition:', error);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -448,8 +548,15 @@ CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
icon: row.icon,
|
||||
sourceCode: row.source_code,
|
||||
credentialCode: row.credential_code,
|
||||
documentation: row.documentation_markdown,
|
||||
documentationMarkdown: row.documentation_markdown,
|
||||
documentationUrl: row.documentation_url,
|
||||
documentationTitle: row.documentation_title,
|
||||
operations: row.operations ? JSON.parse(row.operations) : null,
|
||||
apiMethods: row.api_methods ? JSON.parse(row.api_methods) : null,
|
||||
documentationExamples: row.documentation_examples ? JSON.parse(row.documentation_examples) : null,
|
||||
templates: row.templates ? JSON.parse(row.templates) : null,
|
||||
relatedResources: row.related_resources ? JSON.parse(row.related_resources) : null,
|
||||
requiredScopes: row.required_scopes ? JSON.parse(row.required_scopes) : null,
|
||||
exampleWorkflow: row.example_workflow ? JSON.parse(row.example_workflow) : null,
|
||||
exampleParameters: row.example_parameters ? JSON.parse(row.example_parameters) : null,
|
||||
propertiesSchema: row.properties_schema ? JSON.parse(row.properties_schema) : null,
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
import { NodeSourceInfo } from '../utils/node-source-extractor';
|
||||
import { logger } from '../utils/logger';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
export interface StoredNode {
|
||||
id: string;
|
||||
nodeType: string;
|
||||
name: string;
|
||||
packageName: string;
|
||||
displayName?: string;
|
||||
description?: string;
|
||||
codeHash: string;
|
||||
codeLength: number;
|
||||
sourceLocation: string;
|
||||
hasCredentials: boolean;
|
||||
extractedAt: Date;
|
||||
updatedAt: Date;
|
||||
sourceCode?: string;
|
||||
credentialCode?: string;
|
||||
packageInfo?: any;
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface NodeSearchQuery {
|
||||
query?: string;
|
||||
packageName?: string;
|
||||
nodeType?: string;
|
||||
hasCredentials?: boolean;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export class NodeStorageService {
|
||||
private nodes: Map<string, StoredNode> = new Map();
|
||||
private nodesByPackage: Map<string, Set<string>> = new Map();
|
||||
private searchIndex: Map<string, Set<string>> = new Map();
|
||||
|
||||
/**
|
||||
* Store a node in the database
|
||||
*/
|
||||
async storeNode(nodeInfo: NodeSourceInfo): Promise<StoredNode> {
|
||||
const codeHash = crypto.createHash('sha256').update(nodeInfo.sourceCode).digest('hex');
|
||||
|
||||
// Parse display name and description from source if possible
|
||||
const displayName = this.extractDisplayName(nodeInfo.sourceCode);
|
||||
const description = this.extractDescription(nodeInfo.sourceCode);
|
||||
|
||||
const storedNode: StoredNode = {
|
||||
id: crypto.randomUUID(),
|
||||
nodeType: nodeInfo.nodeType,
|
||||
name: nodeInfo.nodeType.split('.').pop() || nodeInfo.nodeType,
|
||||
packageName: nodeInfo.nodeType.split('.')[0] || 'unknown',
|
||||
displayName,
|
||||
description,
|
||||
codeHash,
|
||||
codeLength: nodeInfo.sourceCode.length,
|
||||
sourceLocation: nodeInfo.location,
|
||||
hasCredentials: !!nodeInfo.credentialCode,
|
||||
extractedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
sourceCode: nodeInfo.sourceCode,
|
||||
credentialCode: nodeInfo.credentialCode,
|
||||
packageInfo: nodeInfo.packageInfo,
|
||||
};
|
||||
|
||||
// Store in memory (replace with real DB)
|
||||
this.nodes.set(nodeInfo.nodeType, storedNode);
|
||||
|
||||
// Update package index
|
||||
if (!this.nodesByPackage.has(storedNode.packageName)) {
|
||||
this.nodesByPackage.set(storedNode.packageName, new Set());
|
||||
}
|
||||
this.nodesByPackage.get(storedNode.packageName)!.add(nodeInfo.nodeType);
|
||||
|
||||
// Update search index
|
||||
this.updateSearchIndex(storedNode);
|
||||
|
||||
logger.info(`Stored node: ${nodeInfo.nodeType} (${codeHash.substring(0, 8)}...)`);
|
||||
return storedNode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for nodes
|
||||
*/
|
||||
async searchNodes(query: NodeSearchQuery): Promise<StoredNode[]> {
|
||||
let results: StoredNode[] = [];
|
||||
|
||||
if (query.query) {
|
||||
// Text search
|
||||
const searchTerms = query.query.toLowerCase().split(' ');
|
||||
const matchingNodeTypes = new Set<string>();
|
||||
|
||||
for (const term of searchTerms) {
|
||||
const matches = this.searchIndex.get(term) || new Set();
|
||||
matches.forEach(nodeType => matchingNodeTypes.add(nodeType));
|
||||
}
|
||||
|
||||
results = Array.from(matchingNodeTypes)
|
||||
.map(nodeType => this.nodes.get(nodeType)!)
|
||||
.filter(Boolean);
|
||||
} else {
|
||||
// Get all nodes
|
||||
results = Array.from(this.nodes.values());
|
||||
}
|
||||
|
||||
// Apply filters
|
||||
if (query.packageName) {
|
||||
results = results.filter(node => node.packageName === query.packageName);
|
||||
}
|
||||
|
||||
if (query.nodeType) {
|
||||
results = results.filter(node => node.nodeType.includes(query.nodeType!));
|
||||
}
|
||||
|
||||
if (query.hasCredentials !== undefined) {
|
||||
results = results.filter(node => node.hasCredentials === query.hasCredentials);
|
||||
}
|
||||
|
||||
// Apply pagination
|
||||
const offset = query.offset || 0;
|
||||
const limit = query.limit || 50;
|
||||
|
||||
return results.slice(offset, offset + limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get node by type
|
||||
*/
|
||||
async getNode(nodeType: string): Promise<StoredNode | null> {
|
||||
return this.nodes.get(nodeType) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all packages
|
||||
*/
|
||||
async getPackages(): Promise<Array<{ name: string; nodeCount: number }>> {
|
||||
return Array.from(this.nodesByPackage.entries()).map(([name, nodes]) => ({
|
||||
name,
|
||||
nodeCount: nodes.size,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk store nodes
|
||||
*/
|
||||
async bulkStoreNodes(nodeInfos: NodeSourceInfo[]): Promise<{
|
||||
stored: number;
|
||||
failed: number;
|
||||
errors: Array<{ nodeType: string; error: string }>;
|
||||
}> {
|
||||
const results = {
|
||||
stored: 0,
|
||||
failed: 0,
|
||||
errors: [] as Array<{ nodeType: string; error: string }>,
|
||||
};
|
||||
|
||||
for (const nodeInfo of nodeInfos) {
|
||||
try {
|
||||
await this.storeNode(nodeInfo);
|
||||
results.stored++;
|
||||
} catch (error) {
|
||||
results.failed++;
|
||||
results.errors.push({
|
||||
nodeType: nodeInfo.nodeType,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate statistics
|
||||
*/
|
||||
async getStatistics(): Promise<{
|
||||
totalNodes: number;
|
||||
totalPackages: number;
|
||||
totalCodeSize: number;
|
||||
nodesWithCredentials: number;
|
||||
averageNodeSize: number;
|
||||
packageDistribution: Array<{ package: string; count: number }>;
|
||||
}> {
|
||||
const nodes = Array.from(this.nodes.values());
|
||||
const totalCodeSize = nodes.reduce((sum, node) => sum + node.codeLength, 0);
|
||||
const nodesWithCredentials = nodes.filter(node => node.hasCredentials).length;
|
||||
|
||||
const packageDistribution = Array.from(this.nodesByPackage.entries())
|
||||
.map(([pkg, nodeSet]) => ({ package: pkg, count: nodeSet.size }))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
|
||||
return {
|
||||
totalNodes: nodes.length,
|
||||
totalPackages: this.nodesByPackage.size,
|
||||
totalCodeSize,
|
||||
nodesWithCredentials,
|
||||
averageNodeSize: nodes.length > 0 ? Math.round(totalCodeSize / nodes.length) : 0,
|
||||
packageDistribution,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract display name from source code
|
||||
*/
|
||||
private extractDisplayName(sourceCode: string): string | undefined {
|
||||
const match = sourceCode.match(/displayName:\s*["'`]([^"'`]+)["'`]/);
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract description from source code
|
||||
*/
|
||||
private extractDescription(sourceCode: string): string | undefined {
|
||||
const match = sourceCode.match(/description:\s*["'`]([^"'`]+)["'`]/);
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update search index
|
||||
*/
|
||||
private updateSearchIndex(node: StoredNode): void {
|
||||
// Index by name parts
|
||||
const nameParts = node.name.toLowerCase().split(/(?=[A-Z])|[._-]/).filter(Boolean);
|
||||
for (const part of nameParts) {
|
||||
if (!this.searchIndex.has(part)) {
|
||||
this.searchIndex.set(part, new Set());
|
||||
}
|
||||
this.searchIndex.get(part)!.add(node.nodeType);
|
||||
}
|
||||
|
||||
// Index by display name
|
||||
if (node.displayName) {
|
||||
const displayParts = node.displayName.toLowerCase().split(/\s+/);
|
||||
for (const part of displayParts) {
|
||||
if (!this.searchIndex.has(part)) {
|
||||
this.searchIndex.set(part, new Set());
|
||||
}
|
||||
this.searchIndex.get(part)!.add(node.nodeType);
|
||||
}
|
||||
}
|
||||
|
||||
// Index by package name
|
||||
const pkgParts = node.packageName.toLowerCase().split(/[.-]/);
|
||||
for (const part of pkgParts) {
|
||||
if (!this.searchIndex.has(part)) {
|
||||
this.searchIndex.set(part, new Set());
|
||||
}
|
||||
this.searchIndex.get(part)!.add(node.nodeType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all nodes for database import
|
||||
*/
|
||||
async exportForDatabase(): Promise<{
|
||||
nodes: StoredNode[];
|
||||
metadata: {
|
||||
exportedAt: Date;
|
||||
totalNodes: number;
|
||||
totalPackages: number;
|
||||
};
|
||||
}> {
|
||||
const nodes = Array.from(this.nodes.values());
|
||||
|
||||
return {
|
||||
nodes,
|
||||
metadata: {
|
||||
exportedAt: new Date(),
|
||||
totalNodes: nodes.length,
|
||||
totalPackages: this.nodesByPackage.size,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,410 +0,0 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import * as crypto from 'crypto';
|
||||
import { NodeSourceInfo } from '../utils/node-source-extractor';
|
||||
import { StoredNode, NodeSearchQuery } from './node-storage-service';
|
||||
import { logger } from '../utils/logger';
|
||||
|
||||
export class SQLiteStorageService {
|
||||
private db: Database.Database;
|
||||
private readonly dbPath: string;
|
||||
|
||||
constructor(dbPath?: string) {
|
||||
this.dbPath = dbPath || process.env.NODE_DB_PATH || path.join(process.cwd(), 'data', 'nodes.db');
|
||||
|
||||
// Ensure data directory exists
|
||||
const dataDir = path.dirname(this.dbPath);
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
this.db = new Database(this.dbPath, {
|
||||
verbose: process.env.NODE_ENV === 'development' ? (msg: unknown) => logger.debug(String(msg)) : undefined
|
||||
});
|
||||
|
||||
// Enable WAL mode for better performance
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
|
||||
this.initializeDatabase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize database with schema
|
||||
*/
|
||||
private initializeDatabase(): void {
|
||||
try {
|
||||
const schema = `
|
||||
-- Main nodes table
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_type TEXT UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
package_name TEXT NOT NULL,
|
||||
display_name TEXT,
|
||||
description TEXT,
|
||||
code_hash TEXT NOT NULL,
|
||||
code_length INTEGER NOT NULL,
|
||||
source_location TEXT NOT NULL,
|
||||
source_code TEXT NOT NULL,
|
||||
credential_code TEXT,
|
||||
package_info TEXT, -- JSON
|
||||
has_credentials INTEGER DEFAULT 0,
|
||||
extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_package_name ON nodes(package_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_code_hash ON nodes(code_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
|
||||
|
||||
-- Full Text Search virtual table for node search
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
|
||||
node_type,
|
||||
name,
|
||||
display_name,
|
||||
description,
|
||||
package_name,
|
||||
content=nodes,
|
||||
content_rowid=id
|
||||
);
|
||||
|
||||
-- Triggers to keep FTS in sync
|
||||
CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes
|
||||
BEGIN
|
||||
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
|
||||
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes
|
||||
BEGIN
|
||||
DELETE FROM nodes_fts WHERE rowid = old.id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes
|
||||
BEGIN
|
||||
DELETE FROM nodes_fts WHERE rowid = old.id;
|
||||
INSERT INTO nodes_fts(rowid, node_type, name, display_name, description, package_name)
|
||||
VALUES (new.id, new.node_type, new.name, new.display_name, new.description, new.package_name);
|
||||
END;
|
||||
|
||||
-- Statistics table for metadata
|
||||
CREATE TABLE IF NOT EXISTS extraction_stats (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
total_nodes INTEGER NOT NULL,
|
||||
total_packages INTEGER NOT NULL,
|
||||
total_code_size INTEGER NOT NULL,
|
||||
nodes_with_credentials INTEGER NOT NULL,
|
||||
extraction_date DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`;
|
||||
|
||||
this.db.exec(schema);
|
||||
logger.info('Database initialized successfully');
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize database:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a node in the database
|
||||
*/
|
||||
async storeNode(nodeInfo: NodeSourceInfo): Promise<StoredNode> {
|
||||
const codeHash = crypto.createHash('sha256').update(nodeInfo.sourceCode).digest('hex');
|
||||
|
||||
// Parse display name and description from source
|
||||
const displayName = this.extractDisplayName(nodeInfo.sourceCode);
|
||||
const description = this.extractDescription(nodeInfo.sourceCode);
|
||||
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
node_type, name, package_name, display_name, description,
|
||||
code_hash, code_length, source_location, source_code,
|
||||
credential_code, package_info, has_credentials,
|
||||
updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP
|
||||
)
|
||||
`);
|
||||
|
||||
const name = nodeInfo.nodeType.split('.').pop() || nodeInfo.nodeType;
|
||||
const packageName = nodeInfo.nodeType.split('.')[0] || 'unknown';
|
||||
|
||||
const result = stmt.run(
|
||||
nodeInfo.nodeType,
|
||||
name,
|
||||
packageName,
|
||||
displayName || null,
|
||||
description || null,
|
||||
codeHash,
|
||||
nodeInfo.sourceCode.length,
|
||||
nodeInfo.location,
|
||||
nodeInfo.sourceCode,
|
||||
nodeInfo.credentialCode || null,
|
||||
nodeInfo.packageInfo ? JSON.stringify(nodeInfo.packageInfo) : null,
|
||||
nodeInfo.credentialCode ? 1 : 0
|
||||
);
|
||||
|
||||
logger.info(`Stored node: ${nodeInfo.nodeType} (${codeHash.substring(0, 8)}...)`);
|
||||
|
||||
return {
|
||||
id: String(result.lastInsertRowid),
|
||||
nodeType: nodeInfo.nodeType,
|
||||
name,
|
||||
packageName,
|
||||
displayName,
|
||||
description,
|
||||
codeHash,
|
||||
codeLength: nodeInfo.sourceCode.length,
|
||||
sourceLocation: nodeInfo.location,
|
||||
hasCredentials: !!nodeInfo.credentialCode,
|
||||
extractedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
sourceCode: nodeInfo.sourceCode,
|
||||
credentialCode: nodeInfo.credentialCode,
|
||||
packageInfo: nodeInfo.packageInfo
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for nodes using FTS
|
||||
*/
|
||||
async searchNodes(query: NodeSearchQuery): Promise<StoredNode[]> {
|
||||
let sql = `
|
||||
SELECT DISTINCT n.*
|
||||
FROM nodes n
|
||||
`;
|
||||
|
||||
const params: any[] = [];
|
||||
const conditions: string[] = [];
|
||||
|
||||
if (query.query) {
|
||||
// Use FTS for text search
|
||||
sql += ` JOIN nodes_fts fts ON n.id = fts.rowid`;
|
||||
conditions.push(`nodes_fts MATCH ?`);
|
||||
// Convert search query to FTS syntax (prefix search)
|
||||
const ftsQuery = query.query.split(' ')
|
||||
.map(term => `${term}*`)
|
||||
.join(' ');
|
||||
params.push(ftsQuery);
|
||||
}
|
||||
|
||||
if (query.packageName) {
|
||||
conditions.push(`n.package_name = ?`);
|
||||
params.push(query.packageName);
|
||||
}
|
||||
|
||||
if (query.nodeType) {
|
||||
conditions.push(`n.node_type LIKE ?`);
|
||||
params.push(`%${query.nodeType}%`);
|
||||
}
|
||||
|
||||
if (query.hasCredentials !== undefined) {
|
||||
conditions.push(`n.has_credentials = ?`);
|
||||
params.push(query.hasCredentials ? 1 : 0);
|
||||
}
|
||||
|
||||
if (conditions.length > 0) {
|
||||
sql += ` WHERE ${conditions.join(' AND ')}`;
|
||||
}
|
||||
|
||||
sql += ` ORDER BY n.name`;
|
||||
|
||||
if (query.limit) {
|
||||
sql += ` LIMIT ?`;
|
||||
params.push(query.limit);
|
||||
|
||||
if (query.offset) {
|
||||
sql += ` OFFSET ?`;
|
||||
params.push(query.offset);
|
||||
}
|
||||
}
|
||||
|
||||
const stmt = this.db.prepare(sql);
|
||||
const rows = stmt.all(...params);
|
||||
|
||||
return rows.map(row => this.rowToStoredNode(row));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get node by type
|
||||
*/
|
||||
async getNode(nodeType: string): Promise<StoredNode | null> {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT * FROM nodes WHERE node_type = ?
|
||||
`);
|
||||
|
||||
const row = stmt.get(nodeType);
|
||||
return row ? this.rowToStoredNode(row) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all packages
|
||||
*/
|
||||
async getPackages(): Promise<Array<{ name: string; nodeCount: number }>> {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT package_name as name, COUNT(*) as nodeCount
|
||||
FROM nodes
|
||||
GROUP BY package_name
|
||||
ORDER BY nodeCount DESC
|
||||
`);
|
||||
|
||||
return stmt.all() as Array<{ name: string; nodeCount: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk store nodes (used for database rebuild)
|
||||
*/
|
||||
async bulkStoreNodes(nodeInfos: NodeSourceInfo[]): Promise<{
|
||||
stored: number;
|
||||
failed: number;
|
||||
errors: Array<{ nodeType: string; error: string }>;
|
||||
}> {
|
||||
const results = {
|
||||
stored: 0,
|
||||
failed: 0,
|
||||
errors: [] as Array<{ nodeType: string; error: string }>
|
||||
};
|
||||
|
||||
// Use transaction for bulk insert
|
||||
const insertMany = this.db.transaction((nodes: NodeSourceInfo[]) => {
|
||||
for (const nodeInfo of nodes) {
|
||||
try {
|
||||
this.storeNode(nodeInfo);
|
||||
results.stored++;
|
||||
} catch (error) {
|
||||
results.failed++;
|
||||
results.errors.push({
|
||||
nodeType: nodeInfo.nodeType,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(nodeInfos);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
async getStatistics(): Promise<{
|
||||
totalNodes: number;
|
||||
totalPackages: number;
|
||||
totalCodeSize: number;
|
||||
nodesWithCredentials: number;
|
||||
averageNodeSize: number;
|
||||
packageDistribution: Array<{ package: string; count: number }>;
|
||||
}> {
|
||||
const stats = this.db.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as totalNodes,
|
||||
COUNT(DISTINCT package_name) as totalPackages,
|
||||
SUM(code_length) as totalCodeSize,
|
||||
SUM(has_credentials) as nodesWithCredentials
|
||||
FROM nodes
|
||||
`).get() as any;
|
||||
|
||||
const packageDist = this.db.prepare(`
|
||||
SELECT package_name as package, COUNT(*) as count
|
||||
FROM nodes
|
||||
GROUP BY package_name
|
||||
ORDER BY count DESC
|
||||
`).all() as Array<{ package: string; count: number }>;
|
||||
|
||||
return {
|
||||
totalNodes: stats.totalNodes || 0,
|
||||
totalPackages: stats.totalPackages || 0,
|
||||
totalCodeSize: stats.totalCodeSize || 0,
|
||||
nodesWithCredentials: stats.nodesWithCredentials || 0,
|
||||
averageNodeSize: stats.totalNodes > 0 ? Math.round(stats.totalCodeSize / stats.totalNodes) : 0,
|
||||
packageDistribution: packageDist
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild entire database
|
||||
*/
|
||||
async rebuildDatabase(): Promise<void> {
|
||||
logger.info('Starting database rebuild...');
|
||||
|
||||
// Clear existing data
|
||||
this.db.exec('DELETE FROM nodes');
|
||||
this.db.exec('DELETE FROM extraction_stats');
|
||||
|
||||
logger.info('Database cleared for rebuild');
|
||||
}
|
||||
|
||||
/**
|
||||
* Save extraction statistics
|
||||
*/
|
||||
async saveExtractionStats(stats: {
|
||||
totalNodes: number;
|
||||
totalPackages: number;
|
||||
totalCodeSize: number;
|
||||
nodesWithCredentials: number;
|
||||
}): Promise<void> {
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO extraction_stats (
|
||||
total_nodes, total_packages, total_code_size, nodes_with_credentials
|
||||
) VALUES (?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
stmt.run(
|
||||
stats.totalNodes,
|
||||
stats.totalPackages,
|
||||
stats.totalCodeSize,
|
||||
stats.nodesWithCredentials
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert database row to StoredNode
|
||||
*/
|
||||
private rowToStoredNode(row: any): StoredNode {
|
||||
return {
|
||||
id: String(row.id),
|
||||
nodeType: row.node_type,
|
||||
name: row.name,
|
||||
packageName: row.package_name,
|
||||
displayName: row.display_name,
|
||||
description: row.description,
|
||||
codeHash: row.code_hash,
|
||||
codeLength: row.code_length,
|
||||
sourceLocation: row.source_location,
|
||||
hasCredentials: row.has_credentials === 1,
|
||||
extractedAt: new Date(row.extracted_at),
|
||||
updatedAt: new Date(row.updated_at),
|
||||
sourceCode: row.source_code,
|
||||
credentialCode: row.credential_code,
|
||||
packageInfo: row.package_info ? JSON.parse(row.package_info) : undefined
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract display name from source code
|
||||
*/
|
||||
private extractDisplayName(sourceCode: string): string | undefined {
|
||||
const match = sourceCode.match(/displayName:\s*["'`]([^"'`]+)["'`]/);
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract description from source code
|
||||
*/
|
||||
private extractDescription(sourceCode: string): string | undefined {
|
||||
const match = sourceCode.match(/description:\s*["'`]([^"'`]+)["'`]/);
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user