From 74f018049dfb961fe567a271ba2564623bbf91fd Mon Sep 17 00:00:00 2001 From: czlonkowski <56956555+czlonkowski@users.noreply.github.com> Date: Sun, 6 Jul 2025 13:11:38 +0200 Subject: [PATCH] feat: add template sanitization to remove API tokens from workflow templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add TemplateSanitizer utility class for detecting and replacing API tokens - Update template repository to automatically sanitize on save - Add sanitize:templates command to clean existing templates - Uses pattern matching to detect various API token formats - Fixes GitHub push protection blocking database updates Note: Database will be updated separately after code is deployed 🤖 Generated with Claude Code Co-Authored-By: Claude --- package.json | 1 + src/scripts/sanitize-templates.ts | 69 ++++++++++++ src/templates/template-repository.ts | 23 +++- src/utils/template-sanitizer.ts | 157 +++++++++++++++++++++++++++ 4 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 src/scripts/sanitize-templates.ts create mode 100644 src/utils/template-sanitizer.ts diff --git a/package.json b/package.json index 3618f48..5921a83 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:tools-documentation": "node dist/scripts/test-tools-documentation.js", "test:mcp:update-partial": "node dist/scripts/test-mcp-n8n-update-partial.js", "test:update-partial:debug": "node dist/scripts/test-update-partial-debug.js", + "sanitize:templates": "node dist/scripts/sanitize-templates.js", "db:rebuild": "node dist/scripts/rebuild-database.js", "db:init": "node -e \"new (require('./dist/services/sqlite-storage-service').SQLiteStorageService)(); console.log('Database initialized')\"", "docs:rebuild": "ts-node src/scripts/rebuild-database.ts", diff --git a/src/scripts/sanitize-templates.ts b/src/scripts/sanitize-templates.ts new file mode 100644 index 0000000..2946e9c --- /dev/null +++ b/src/scripts/sanitize-templates.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node +import { createDatabaseAdapter } from '../database/database-adapter'; +import { logger } from '../utils/logger'; +import { TemplateSanitizer } from '../utils/template-sanitizer'; + +async function sanitizeTemplates() { + console.log('🧹 Sanitizing workflow templates in database...\n'); + + const db = await createDatabaseAdapter('./data/nodes.db'); + const sanitizer = new TemplateSanitizer(); + + try { + // Get all templates + const templates = db.prepare('SELECT id, name, workflow_json FROM templates').all() as any[]; + console.log(`Found ${templates.length} templates to check\n`); + + let sanitizedCount = 0; + const problematicTemplates: any[] = []; + + for (const template of templates) { + const originalWorkflow = JSON.parse(template.workflow_json); + const { sanitized: sanitizedWorkflow, wasModified } = sanitizer.sanitizeWorkflow(originalWorkflow); + + if (wasModified) { + // Get detected tokens for reporting + const detectedTokens = sanitizer.detectTokens(originalWorkflow); + + // Update the template with sanitized version + const stmt = db.prepare('UPDATE templates SET workflow_json = ? WHERE id = ?'); + stmt.run(JSON.stringify(sanitizedWorkflow), template.id); + + sanitizedCount++; + problematicTemplates.push({ + id: template.id, + name: template.name, + tokens: detectedTokens + }); + + console.log(`✅ Sanitized template ${template.id}: ${template.name}`); + detectedTokens.forEach(token => { + console.log(` - Found: ${token.substring(0, 20)}...`); + }); + } + } + + console.log(`\n📊 Summary:`); + console.log(` Total templates: ${templates.length}`); + console.log(` Sanitized: ${sanitizedCount}`); + + if (problematicTemplates.length > 0) { + console.log(`\n⚠️ Templates that contained API tokens:`); + problematicTemplates.forEach(t => { + console.log(` - ${t.id}: ${t.name}`); + }); + } + + console.log('\n✨ Sanitization complete!'); + } catch (error) { + console.error('❌ Error sanitizing templates:', error); + process.exit(1); + } finally { + db.close(); + } +} + +// Run if called directly +if (require.main === module) { + sanitizeTemplates().catch(console.error); +} \ No newline at end of file diff --git a/src/templates/template-repository.ts b/src/templates/template-repository.ts index 4f1e6ac..15c43c6 100644 --- a/src/templates/template-repository.ts +++ b/src/templates/template-repository.ts @@ -1,6 +1,7 @@ import { DatabaseAdapter } from '../database/database-adapter'; import { TemplateWorkflow, TemplateDetail } from './template-fetcher'; import { logger } from '../utils/logger'; +import { TemplateSanitizer } from '../utils/template-sanitizer'; export interface StoredTemplate { id: number; @@ -21,7 +22,11 @@ export interface StoredTemplate { } export class TemplateRepository { - constructor(private db: DatabaseAdapter) {} + private sanitizer: TemplateSanitizer; + + constructor(private db: DatabaseAdapter) { + this.sanitizer = new TemplateSanitizer(); + } /** * Save a template to the database @@ -41,6 +46,20 @@ export class TemplateRepository { // Build URL const url = `https://n8n.io/workflows/${workflow.id}`; + // Sanitize the workflow to remove API tokens + const { sanitized: sanitizedWorkflow, wasModified } = this.sanitizer.sanitizeWorkflow(detail.workflow); + + // Log if we sanitized any tokens + if (wasModified) { + const detectedTokens = this.sanitizer.detectTokens(detail.workflow); + logger.warn(`Sanitized API tokens in template ${workflow.id}: ${workflow.name}`, { + templateId: workflow.id, + templateName: workflow.name, + tokensFound: detectedTokens.length, + tokenPreviews: detectedTokens.map(t => t.substring(0, 20) + '...') + }); + } + stmt.run( workflow.id, workflow.id, @@ -50,7 +69,7 @@ export class TemplateRepository { workflow.user.username, workflow.user.verified ? 1 : 0, JSON.stringify(nodeTypes), - JSON.stringify(detail.workflow), + JSON.stringify(sanitizedWorkflow), JSON.stringify(categories), workflow.totalViews || 0, workflow.createdAt, diff --git a/src/utils/template-sanitizer.ts b/src/utils/template-sanitizer.ts new file mode 100644 index 0000000..b9af9d9 --- /dev/null +++ b/src/utils/template-sanitizer.ts @@ -0,0 +1,157 @@ +import { logger } from './logger'; + +/** + * Configuration for template sanitization + */ +export interface SanitizerConfig { + problematicTokens: string[]; + tokenPatterns: RegExp[]; + replacements: Map; +} + +/** + * Default sanitizer configuration + */ +export const defaultSanitizerConfig: SanitizerConfig = { + problematicTokens: [ + // Specific tokens can be added here if needed + ], + tokenPatterns: [ + /apify_api_[A-Za-z0-9]+/g, + /sk-[A-Za-z0-9]+/g, // OpenAI tokens + /Bearer\s+[A-Za-z0-9\-._~+\/]+=*/g // Generic bearer tokens + ], + replacements: new Map([ + ['apify_api_', 'apify_api_YOUR_TOKEN_HERE'], + ['sk-', 'sk-YOUR_OPENAI_KEY_HERE'], + ['Bearer ', 'Bearer YOUR_TOKEN_HERE'] + ]) +}; + +/** + * Template sanitizer for removing API tokens from workflow templates + */ +export class TemplateSanitizer { + constructor(private config: SanitizerConfig = defaultSanitizerConfig) {} + + /** + * Add a new problematic token to sanitize + */ + addProblematicToken(token: string): void { + if (!this.config.problematicTokens.includes(token)) { + this.config.problematicTokens.push(token); + logger.info(`Added problematic token to sanitizer: ${token.substring(0, 10)}...`); + } + } + + /** + * Add a new token pattern to detect + */ + addTokenPattern(pattern: RegExp, replacement: string): void { + this.config.tokenPatterns.push(pattern); + const prefix = pattern.source.match(/^([^[]+)/)?.[1] || ''; + if (prefix) { + this.config.replacements.set(prefix, replacement); + } + } + + /** + * Sanitize a workflow object + */ + sanitizeWorkflow(workflow: any): { sanitized: any; wasModified: boolean } { + const original = JSON.stringify(workflow); + const sanitized = this.sanitizeObject(workflow); + const wasModified = JSON.stringify(sanitized) !== original; + + return { sanitized, wasModified }; + } + + /** + * Check if a workflow needs sanitization + */ + needsSanitization(workflow: any): boolean { + const workflowStr = JSON.stringify(workflow); + + // Check for known problematic tokens + for (const token of this.config.problematicTokens) { + if (workflowStr.includes(token)) { + return true; + } + } + + // Check for token patterns + for (const pattern of this.config.tokenPatterns) { + pattern.lastIndex = 0; // Reset regex state + if (pattern.test(workflowStr)) { + return true; + } + } + + return false; + } + + /** + * Get list of detected tokens in a workflow + */ + detectTokens(workflow: any): string[] { + const workflowStr = JSON.stringify(workflow); + const detectedTokens: string[] = []; + + // Check for known problematic tokens + for (const token of this.config.problematicTokens) { + if (workflowStr.includes(token)) { + detectedTokens.push(token); + } + } + + // Check for token patterns + for (const pattern of this.config.tokenPatterns) { + pattern.lastIndex = 0; // Reset regex state + const matches = workflowStr.match(pattern); + if (matches) { + detectedTokens.push(...matches); + } + } + + return [...new Set(detectedTokens)]; // Remove duplicates + } + + private sanitizeObject(obj: any): any { + if (typeof obj === 'string') { + return this.replaceTokens(obj); + } else if (Array.isArray(obj)) { + return obj.map(item => this.sanitizeObject(item)); + } else if (obj && typeof obj === 'object') { + const result: any = {}; + for (const key in obj) { + result[key] = this.sanitizeObject(obj[key]); + } + return result; + } + return obj; + } + + private replaceTokens(str: string): string { + let result = str; + + // Replace known problematic tokens + this.config.problematicTokens.forEach(token => { + result = result.replace(new RegExp(token, 'g'), 'YOUR_API_TOKEN_HERE'); + }); + + // Replace pattern-matched tokens + this.config.tokenPatterns.forEach(pattern => { + result = result.replace(pattern, (match) => { + // Find the best replacement based on prefix + for (const [prefix, replacement] of this.config.replacements) { + if (match.startsWith(prefix)) { + return replacement; + } + } + return 'YOUR_TOKEN_HERE'; + }); + }); + + return result; + } +} \ No newline at end of file