feat: expand template window from 6 months to 1 year and add robust fetcher

- Changed template time window from 6 months to 1 year
- Added robust incremental template fetcher that saves as it goes
- Successfully fetched and saved 399 templates (up from 199)
- Added npm script 'fetch:templates:robust' for better error handling
- Templates now save incrementally to prevent data loss on errors

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
czlonkowski
2025-06-20 00:52:26 +02:00
parent 12d23d65a0
commit 48fd319a1b
5 changed files with 138 additions and 7 deletions

View File

@@ -42,8 +42,8 @@ export class TemplateFetcher {
private readonly pageSize = 100;
async fetchTemplates(progressCallback?: (current: number, total: number) => void): Promise<TemplateWorkflow[]> {
const sixMonthsAgo = new Date();
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
const oneYearAgo = new Date();
oneYearAgo.setMonth(oneYearAgo.getMonth() - 12);
const allTemplates: TemplateWorkflow[] = [];
let page = 1;
@@ -66,13 +66,13 @@ export class TemplateFetcher {
// Filter templates by date
const recentTemplates = workflows.filter((w: TemplateWorkflow) => {
const createdDate = new Date(w.createdAt);
return createdDate >= sixMonthsAgo;
return createdDate >= oneYearAgo;
});
// If we hit templates older than 6 months, stop fetching
// If we hit templates older than 1 year, stop fetching
if (recentTemplates.length < workflows.length) {
hasMore = false;
logger.info(`Reached templates older than 6 months at page ${page}`);
logger.info(`Reached templates older than 1 year at page ${page}`);
}
allTemplates.push(...recentTemplates);
@@ -98,7 +98,7 @@ export class TemplateFetcher {
}
}
logger.info(`Fetched ${allTemplates.length} templates from last 6 months`);
logger.info(`Fetched ${allTemplates.length} templates from last year`);
return allTemplates;
}

View File

@@ -112,7 +112,7 @@ export class TemplateService {
progressCallback?.('Fetching template list', current, total);
});
logger.info(`Found ${templates.length} templates from last 6 months`);
logger.info(`Found ${templates.length} templates from last year`);
// Fetch details for each template
logger.info('Fetching template details');