diff --git a/data/nodes.db b/data/nodes.db index 119614a..0c9fc84 100644 Binary files a/data/nodes.db and b/data/nodes.db differ diff --git a/src/scripts/sanitize-templates.ts b/src/scripts/sanitize-templates.ts index 988d724..6b87165 100644 --- a/src/scripts/sanitize-templates.ts +++ b/src/scripts/sanitize-templates.ts @@ -2,32 +2,50 @@ import { createDatabaseAdapter } from '../database/database-adapter'; import { logger } from '../utils/logger'; import { TemplateSanitizer } from '../utils/template-sanitizer'; +import { gunzipSync, gzipSync } from 'zlib'; async function sanitizeTemplates() { console.log('🧹 Sanitizing workflow templates in database...\n'); - + const db = await createDatabaseAdapter('./data/nodes.db'); const sanitizer = new TemplateSanitizer(); - + try { - // Get all templates - const templates = db.prepare('SELECT id, name, workflow_json FROM templates').all() as any[]; + // Get all templates - check both old and new format + const templates = db.prepare('SELECT id, name, workflow_json, workflow_json_compressed FROM templates').all() as any[]; console.log(`Found ${templates.length} templates to check\n`); - + let sanitizedCount = 0; const problematicTemplates: any[] = []; - + for (const template of templates) { - if (!template.workflow_json) { - continue; // Skip templates without workflow data + let originalWorkflow: any = null; + let useCompressed = false; + + // Try compressed format first (newer format) + if (template.workflow_json_compressed) { + try { + const buffer = Buffer.from(template.workflow_json_compressed, 'base64'); + const decompressed = gunzipSync(buffer).toString('utf-8'); + originalWorkflow = JSON.parse(decompressed); + useCompressed = true; + } catch (e) { + console.log(`⚠️ Failed to decompress template ${template.id}, trying uncompressed`); + } } - let originalWorkflow; - try { - originalWorkflow = JSON.parse(template.workflow_json); - } catch (e) { - console.log(`⚠️ Skipping template ${template.id}: Invalid JSON`); - continue; + // Fall back to uncompressed format (deprecated) + if (!originalWorkflow && template.workflow_json) { + try { + originalWorkflow = JSON.parse(template.workflow_json); + } catch (e) { + console.log(`⚠️ Skipping template ${template.id}: Invalid JSON in both formats`); + continue; + } + } + + if (!originalWorkflow) { + continue; // Skip templates without workflow data } const { sanitized: sanitizedWorkflow, wasModified } = sanitizer.sanitizeWorkflow(originalWorkflow); @@ -35,18 +53,24 @@ async function sanitizeTemplates() { if (wasModified) { // Get detected tokens for reporting const detectedTokens = sanitizer.detectTokens(originalWorkflow); - - // Update the template with sanitized version - const stmt = db.prepare('UPDATE templates SET workflow_json = ? WHERE id = ?'); - stmt.run(JSON.stringify(sanitizedWorkflow), template.id); - + + // Update the template with sanitized version in the same format + if (useCompressed) { + const compressed = gzipSync(JSON.stringify(sanitizedWorkflow)).toString('base64'); + const stmt = db.prepare('UPDATE templates SET workflow_json_compressed = ? WHERE id = ?'); + stmt.run(compressed, template.id); + } else { + const stmt = db.prepare('UPDATE templates SET workflow_json = ? WHERE id = ?'); + stmt.run(JSON.stringify(sanitizedWorkflow), template.id); + } + sanitizedCount++; problematicTemplates.push({ id: template.id, name: template.name, tokens: detectedTokens }); - + console.log(`✅ Sanitized template ${template.id}: ${template.name}`); detectedTokens.forEach(token => { console.log(` - Found: ${token.substring(0, 20)}...`);