From 8d1ae278eec198245ad264dc5506365f47f00e31 Mon Sep 17 00:00:00 2001 From: czlonkowski <56956555+czlonkowski@users.noreply.github.com> Date: Fri, 3 Oct 2025 13:36:46 +0200 Subject: [PATCH] fix: optimize search_templates_by_metadata to prevent timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: - search_templates_by_metadata with no filters caused Claude Desktop timeouts - Query loaded ALL templates with metadata_json and decompressed workflows - With 2,646 templates, this caused significant performance issues Solution: - Implement two-phase query optimization: 1. Phase 1: SELECT id only (fast, no workflow data) 2. Phase 2: Fetch full records only for matching IDs (decompress only needed rows) - Prevents loading/decompressing thousands of rows when only 20 are needed Performance Impact: - No filters: Now responds instantly instead of timing out - With filters: Same fast performance, minimal overhead - Only decompresses the exact number of rows needed (limit parameter) Testing: - Tested with no filters: ✅ 2,646 templates, returned 5 in <1s - Tested with complexity filter: ✅ 262 templates, returned 3 in <1s 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- data/nodes.db | Bin 60383232 -> 60383232 bytes src/templates/template-repository.ts | 43 +++++++++++++++++++-------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/data/nodes.db b/data/nodes.db index 78d433d53d159aa9529996274841071d9a647dd3..95893ff9928f58e16cc175f3ab70b2364c6f88da 100644 GIT binary patch delta 3463 zcmWmDX9`n{AtKb+c_7Tej`3w(Q!n-P-BH?;kkNbA}##lz1p6 zCyEgq5Kv%dKtP@j0Rdst1qAG_J0)|jP{G4wg|Y&zAS<*L#tLhNv%*^utcX@5E3y^E zifTo(qFXVnm{u$+wiU;UYsItTTM4X$Rw660mBdPFC9{%SDXf%MDl4^>#!73Yv(j4` ztc+GBE3=iw%4%h^vRgTSy)023P~F zLDpbvh&9w2W(~JSSR*Z4qpZ=^7%RjYYmKwUTNA8_)+B4PHN~20O|zz3Gpw1`ENiwk z$C_)+v*ue1tcBJhYq7P&T52t`mRl>VmDVb2wYA1tYpt`^TN|v6)+TGSwZ;0)`rX=U zZL_vpJFK17E^D{7$J%S{v-VpDtb^7e>#%jiI%*xWj$0?Jlh!Hgv~|WhYn`*sTNkX0 z)+Ot*^@sJRb;Y`B{bl`aU9+xRH>{i1E$g;*$GU6Xv+i3DtcTVk>#_C3`p5d$dTKqh zo?9=hm)0xmwe`k&YrV7HTOX{C)+g(;^~L&XeY3t>Kdk?(pCJl0L7@*BM*<{7A|yr zCS*nyWJNY)M-JpfF62fYArwXt6h$!vqc}>SBub$)%AhRDp*$*}A}XOW zs-P;Wp*m`yCTgKJ>Yy&_p*|X*AsV4EnxH9u^>2peXn~e!h1O_;wrGd;=zxysgwE)K zuIPsD=z*T-h2H3czUYVk7=VEoguxhsp%{kY7=e*6MqxC@AOvGE4&yNa6EO*sF$GgG z4bw3LGcgOZF$Z%o5A(4A3$X}`u>?!849l?sE3pczu?B0g4(qW28?gzSu?4^3cWlKr zY{w4l#4hZ{9_+y zZ(PH5+`vuT!fo8aUEITcJitRd!eczaKlm3<@eI%L0x$6juki+N@ec3t0Uz-RpYa7> z@eSYc1OMS?s1OCRKLQbi&r+Fc5<<7(*}=!!R5p zFcQWnjK&y*U@XRAJSJcwCSfwBU@E3zI%Z%dW??qwU@qoiJ{Djh7GW`#U@4YiIaXjL zR$(>PU@g{RJvLw?HeoZi;5Yn^t=NX`*nyqch27YLz1WBSIDmsVgu^(3qd11+IDwNm zh0{2Lvp9$IxPXhegvjSDiCBn@IEagQh>rwFh(t(?BuI*6NRAXpiBw39G)RkdNRJH2h)l?gEXay% z$c`MyiCoByJjjcD$d3Xjh(aigA}ESt2u5*~KuMHBX_P@(ltXz`Kt)tSWmG{`R6}*t zKuy#_ZPYCfi{OaEf&Cvoa(F(2625r#}?a=`p(FvW=1zph%-O&R* z(F?uN2Yt~G{V@OoF$jY(1Vb?l!!ZIQVT{6Pj6n#-VjRX}0w!V-CSwYwVj8An24-Rw NW(V!loS?-`{|BD939JAB delta 3463 zcmWmD7k=L|jkB=Jy8 zP81_JARzzDfPg$50s_LO3kcX-a7yM}p@N6W3S|XaK~`uhj1|@jXN9*SSP`vAR%9!R z71fGnMYm#DF|AluY%7ix*NSJww-Q(htwdI0D~XlVN@gXuQdlXiR90#$jg{6)XQj6? zSQ)KMR%R=UmDS2-Ww&xzIjvk)ZYz(K*UD!FTluX5Rza(fRoE(G6}5_4#jO%nNvo7q z+A3p}waQuLtqN8}tCCgOs$x~Os#(>o8dgoKmQ~xTW7W0lS@o?3Rzs_i)!1rcHMQu| z%xZ46uv%KJtkzZ=tF6_}YHxM0I$E8q&Q=$ztJTfwZuPKwTD`2^Rv)Xc)z9j04X_4U zgRH^U5NoJ4%o=Wuutr+8Mp>h+F;<8*)*5Gxw_<=S=MZ8 zjy2bsXU(@3SPQL1)?#akwbWW>Ew@%!E3H-5YHN+P)>>z+w>DTCtxeWuYm2qj+GcIH zc33;DUDj@EkG0p@XYIERSO={`)?w?2b<{d$9k)(cC#_S~Y3q!2);edMw=P&0txMKr z>xy;Nx@KLsezShJZdf<1Th?vsj&;|%XWh3RSP!i~tUs+s)?@35_0)Q1J-1$1e_4N9 zFRfSBYwL~m)_P~Xw?0@Otxwix>x=c(`euE%epvrl|62c9KSLC1fK?sd72#atC zj|hl}NQjImh>B>4ju?oEScr`{h>LiLj|51FL`aMzNQz`gjuc3VR7j09NQ-nxj||9& zOvsEZ$ck*pjvUB|T*!?)$cuakMt&4PK@>t^6hToGLvfTqNt8lqltEdPLwQs{MN~p% zR6$i#Lv_?ZP1Hhd)InX;Lwz(rLo`BTG(l7R>fa2_(E=^e3a!xwZP5)aV-40~9oAz5HewStV+*!o8@6Ky zc48NHV-NOXANJz_4&o3F;|Px87>?rvPT~|!;|$K?9M0ncF5(g{;|i|g8m{9v{Ei#A ziCeghJGhH`xQ_>Th(GWr9^o;b;3=NrIbPr|{Ee4*h1Yn4w|Iy5_<)c2gwObbulR=V z_NP$3Fre*_{3p%DgQ5f0%I0TB@ikr4$^5e?B112GW`u@MJx5fAZ^011%@ ziID_JkqpU^0x6LSsgVY0kq+sR0U41AnUMuqkqz0A138fkxseBXkq^Phj{+!&LMV(P zD2iezjuI$|QYeiwD2s9^j|!-WN~nw~sETT+jvAzLq!B~vLcuc@VOu}SL!BkAcbj-j^%))HU!CcJ4d@R61EW%~q9FE1Ac7DYVGtJK5FQZ_5s?rX zQ4kf;5FIfP6R{8*aS#{r5FZJU5Q&f&NstuDkQ^zH5~+|HX^fQqPu%BX^>sD|pO zftsj=+NguNsE7J!fQD#<#%O}3_|?A|nxh3;q7_=B4cej|+M@$Hq7yo!3%a5kx}yhr zq8ECj5Bj1X`eOhFVh{#n2!>)9hGPUq!Wf0o7=sXu#W;+|1Wd#vOvV&U#WYOE49vtV N%nmxBIYEn?{tr*G34j0q diff --git a/src/templates/template-repository.ts b/src/templates/template-repository.ts index 90eeb3d..696e449 100644 --- a/src/templates/template-repository.ts +++ b/src/templates/template-repository.ts @@ -639,7 +639,7 @@ export class TemplateRepository { }, limit: number = 20, offset: number = 0): StoredTemplate[] { const conditions: string[] = ['metadata_json IS NOT NULL']; const params: any[] = []; - + // Build WHERE conditions based on filters with proper parameterization if (filters.category !== undefined) { // Use parameterized LIKE with JSON array search - safe from injection @@ -648,22 +648,22 @@ export class TemplateRepository { const sanitizedCategory = JSON.stringify(filters.category).slice(1, -1); params.push(sanitizedCategory); } - + if (filters.complexity) { conditions.push("json_extract(metadata_json, '$.complexity') = ?"); params.push(filters.complexity); } - + if (filters.maxSetupMinutes !== undefined) { conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) <= ?"); params.push(filters.maxSetupMinutes); } - + if (filters.minSetupMinutes !== undefined) { conditions.push("CAST(json_extract(metadata_json, '$.estimated_setup_minutes') AS INTEGER) >= ?"); params.push(filters.minSetupMinutes); } - + if (filters.requiredService !== undefined) { // Use parameterized LIKE with JSON array search - safe from injection conditions.push("json_extract(metadata_json, '$.required_services') LIKE '%' || ? || '%'"); @@ -671,25 +671,42 @@ export class TemplateRepository { const sanitizedService = JSON.stringify(filters.requiredService).slice(1, -1); params.push(sanitizedService); } - + if (filters.targetAudience !== undefined) { - // Use parameterized LIKE with JSON array search - safe from injection + // Use parameterized LIKE with JSON array search - safe from injection conditions.push("json_extract(metadata_json, '$.target_audience') LIKE '%' || ? || '%'"); // Escape special characters and quotes for JSON string matching const sanitizedAudience = JSON.stringify(filters.targetAudience).slice(1, -1); params.push(sanitizedAudience); } - - const query = ` - SELECT * FROM templates + + // Performance optimization: Use two-phase query to avoid loading large compressed workflows + // during metadata filtering. This prevents timeout when no filters are provided. + // Phase 1: Get IDs only with metadata filtering (fast - no workflow data) + const idsQuery = ` + SELECT id FROM templates WHERE ${conditions.join(' AND ')} ORDER BY views DESC, created_at DESC LIMIT ? OFFSET ? `; - + params.push(limit, offset); - const results = this.db.prepare(query).all(...params) as StoredTemplate[]; - + const ids = this.db.prepare(idsQuery).all(...params) as { id: number }[]; + + if (ids.length === 0) { + logger.debug('Metadata search found 0 results', { filters }); + return []; + } + + // Phase 2: Fetch full records only for matching IDs (only decompress needed rows) + const placeholders = ids.map(() => '?').join(','); + const fullQuery = ` + SELECT * FROM templates + WHERE id IN (${placeholders}) + ORDER BY views DESC, created_at DESC + `; + const results = this.db.prepare(fullQuery).all(...ids.map(r => r.id)) as StoredTemplate[]; + logger.debug(`Metadata search found ${results.length} results`, { filters, count: results.length }); return results.map(t => this.decompressWorkflow(t)); }