feat: expand template window from 6 months to 1 year and add robust fetcher
- Changed template time window from 6 months to 1 year - Added robust incremental template fetcher that saves as it goes - Successfully fetched and saved 399 templates (up from 199) - Added npm script 'fetch:templates:robust' for better error handling - Templates now save incrementally to prevent data loss on errors 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -42,8 +42,8 @@ export class TemplateFetcher {
|
||||
private readonly pageSize = 100;
|
||||
|
||||
async fetchTemplates(progressCallback?: (current: number, total: number) => void): Promise<TemplateWorkflow[]> {
|
||||
const sixMonthsAgo = new Date();
|
||||
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
|
||||
const oneYearAgo = new Date();
|
||||
oneYearAgo.setMonth(oneYearAgo.getMonth() - 12);
|
||||
|
||||
const allTemplates: TemplateWorkflow[] = [];
|
||||
let page = 1;
|
||||
@@ -66,13 +66,13 @@ export class TemplateFetcher {
|
||||
// Filter templates by date
|
||||
const recentTemplates = workflows.filter((w: TemplateWorkflow) => {
|
||||
const createdDate = new Date(w.createdAt);
|
||||
return createdDate >= sixMonthsAgo;
|
||||
return createdDate >= oneYearAgo;
|
||||
});
|
||||
|
||||
// If we hit templates older than 6 months, stop fetching
|
||||
// If we hit templates older than 1 year, stop fetching
|
||||
if (recentTemplates.length < workflows.length) {
|
||||
hasMore = false;
|
||||
logger.info(`Reached templates older than 6 months at page ${page}`);
|
||||
logger.info(`Reached templates older than 1 year at page ${page}`);
|
||||
}
|
||||
|
||||
allTemplates.push(...recentTemplates);
|
||||
@@ -98,7 +98,7 @@ export class TemplateFetcher {
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Fetched ${allTemplates.length} templates from last 6 months`);
|
||||
logger.info(`Fetched ${allTemplates.length} templates from last year`);
|
||||
return allTemplates;
|
||||
}
|
||||
|
||||
|
||||
@@ -112,7 +112,7 @@ export class TemplateService {
|
||||
progressCallback?.('Fetching template list', current, total);
|
||||
});
|
||||
|
||||
logger.info(`Found ${templates.length} templates from last 6 months`);
|
||||
logger.info(`Found ${templates.length} templates from last year`);
|
||||
|
||||
// Fetch details for each template
|
||||
logger.info('Fetching template details');
|
||||
|
||||
Reference in New Issue
Block a user