mirror of
https://github.com/czlonkowski/n8n-mcp.git
synced 2026-01-30 06:22:04 +00:00
320 lines
14 KiB
JavaScript
320 lines
14 KiB
JavaScript
"use strict";
|
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
}
|
|
Object.defineProperty(o, k2, desc);
|
|
}) : (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
o[k2] = m[k];
|
|
}));
|
|
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
}) : function(o, v) {
|
|
o["default"] = v;
|
|
});
|
|
var __importStar = (this && this.__importStar) || (function () {
|
|
var ownKeys = function(o) {
|
|
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
var ar = [];
|
|
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
return ar;
|
|
};
|
|
return ownKeys(o);
|
|
};
|
|
return function (mod) {
|
|
if (mod && mod.__esModule) return mod;
|
|
var result = {};
|
|
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
__setModuleDefault(result, mod);
|
|
return result;
|
|
};
|
|
})();
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
};
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.BatchProcessor = void 0;
|
|
const fs = __importStar(require("fs"));
|
|
const path = __importStar(require("path"));
|
|
const openai_1 = __importDefault(require("openai"));
|
|
const logger_1 = require("../utils/logger");
|
|
const metadata_generator_1 = require("./metadata-generator");
|
|
class BatchProcessor {
|
|
constructor(options) {
|
|
this.client = new openai_1.default({ apiKey: options.apiKey });
|
|
this.generator = new metadata_generator_1.MetadataGenerator(options.apiKey, options.model);
|
|
this.batchSize = options.batchSize || 100;
|
|
this.outputDir = options.outputDir || './temp';
|
|
if (!fs.existsSync(this.outputDir)) {
|
|
fs.mkdirSync(this.outputDir, { recursive: true });
|
|
}
|
|
}
|
|
async processTemplates(templates, progressCallback) {
|
|
const results = new Map();
|
|
const batches = this.createBatches(templates);
|
|
logger_1.logger.info(`Processing ${templates.length} templates in ${batches.length} batches`);
|
|
console.log(`\n📤 Submitting ${batches.length} batch${batches.length > 1 ? 'es' : ''} to OpenAI...`);
|
|
const batchJobs = [];
|
|
for (let i = 0; i < batches.length; i++) {
|
|
const batch = batches[i];
|
|
const batchNum = i + 1;
|
|
try {
|
|
progressCallback?.(`Submitting batch ${batchNum}/${batches.length}`, i * this.batchSize, templates.length);
|
|
const jobPromise = this.submitBatch(batch, `batch_${batchNum}`);
|
|
batchJobs.push({ batchNum, jobPromise, templates: batch });
|
|
console.log(` 📨 Submitted batch ${batchNum}/${batches.length} (${batch.length} templates)`);
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.error(`Error submitting batch ${batchNum}:`, error);
|
|
console.error(` ❌ Failed to submit batch ${batchNum}`);
|
|
}
|
|
}
|
|
console.log(`\n⏳ All batches submitted. Waiting for completion...`);
|
|
console.log(` (Batches process in parallel - this is much faster than sequential processing)`);
|
|
const batchPromises = batchJobs.map(async ({ batchNum, jobPromise, templates: batchTemplates }) => {
|
|
try {
|
|
const completedJob = await jobPromise;
|
|
console.log(`\n📦 Retrieving results for batch ${batchNum}/${batches.length}...`);
|
|
const batchResults = await this.retrieveResults(completedJob);
|
|
logger_1.logger.info(`Retrieved ${batchResults.length} results from batch ${batchNum}`);
|
|
progressCallback?.(`Retrieved batch ${batchNum}/${batches.length}`, Math.min(batchNum * this.batchSize, templates.length), templates.length);
|
|
return { batchNum, results: batchResults };
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.error(`Error processing batch ${batchNum}:`, error);
|
|
console.error(` ❌ Batch ${batchNum} failed:`, error);
|
|
return { batchNum, results: [] };
|
|
}
|
|
});
|
|
const allBatchResults = await Promise.all(batchPromises);
|
|
for (const { batchNum, results: batchResults } of allBatchResults) {
|
|
for (const result of batchResults) {
|
|
results.set(result.templateId, result);
|
|
}
|
|
if (batchResults.length > 0) {
|
|
console.log(` ✅ Merged ${batchResults.length} results from batch ${batchNum}`);
|
|
}
|
|
}
|
|
logger_1.logger.info(`Batch processing complete: ${results.size} results`);
|
|
return results;
|
|
}
|
|
async submitBatch(templates, batchName) {
|
|
const inputFile = await this.createBatchFile(templates, batchName);
|
|
try {
|
|
const uploadedFile = await this.uploadFile(inputFile);
|
|
const batchJob = await this.createBatchJob(uploadedFile.id);
|
|
const monitoringPromise = this.monitorBatchJob(batchJob.id);
|
|
try {
|
|
fs.unlinkSync(inputFile);
|
|
}
|
|
catch { }
|
|
monitoringPromise.then(async (completedJob) => {
|
|
try {
|
|
await this.client.files.del(uploadedFile.id);
|
|
if (completedJob.output_file_id) {
|
|
}
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.warn(`Failed to cleanup files for batch ${batchName}`, error);
|
|
}
|
|
});
|
|
return monitoringPromise;
|
|
}
|
|
catch (error) {
|
|
try {
|
|
fs.unlinkSync(inputFile);
|
|
}
|
|
catch { }
|
|
throw error;
|
|
}
|
|
}
|
|
async processBatch(templates, batchName) {
|
|
const inputFile = await this.createBatchFile(templates, batchName);
|
|
try {
|
|
const uploadedFile = await this.uploadFile(inputFile);
|
|
const batchJob = await this.createBatchJob(uploadedFile.id);
|
|
const completedJob = await this.monitorBatchJob(batchJob.id);
|
|
const results = await this.retrieveResults(completedJob);
|
|
await this.cleanup(inputFile, uploadedFile.id, completedJob.output_file_id);
|
|
return results;
|
|
}
|
|
catch (error) {
|
|
try {
|
|
fs.unlinkSync(inputFile);
|
|
}
|
|
catch { }
|
|
throw error;
|
|
}
|
|
}
|
|
createBatches(templates) {
|
|
const batches = [];
|
|
for (let i = 0; i < templates.length; i += this.batchSize) {
|
|
batches.push(templates.slice(i, i + this.batchSize));
|
|
}
|
|
return batches;
|
|
}
|
|
async createBatchFile(templates, batchName) {
|
|
const filename = path.join(this.outputDir, `${batchName}_${Date.now()}.jsonl`);
|
|
const stream = fs.createWriteStream(filename);
|
|
for (const template of templates) {
|
|
const request = this.generator.createBatchRequest(template);
|
|
stream.write(JSON.stringify(request) + '\n');
|
|
}
|
|
stream.end();
|
|
await new Promise((resolve, reject) => {
|
|
stream.on('finish', () => resolve());
|
|
stream.on('error', reject);
|
|
});
|
|
logger_1.logger.debug(`Created batch file: ${filename} with ${templates.length} requests`);
|
|
return filename;
|
|
}
|
|
async uploadFile(filepath) {
|
|
const file = fs.createReadStream(filepath);
|
|
const uploadedFile = await this.client.files.create({
|
|
file,
|
|
purpose: 'batch'
|
|
});
|
|
logger_1.logger.debug(`Uploaded file: ${uploadedFile.id}`);
|
|
return uploadedFile;
|
|
}
|
|
async createBatchJob(fileId) {
|
|
const batchJob = await this.client.batches.create({
|
|
input_file_id: fileId,
|
|
endpoint: '/v1/chat/completions',
|
|
completion_window: '24h'
|
|
});
|
|
logger_1.logger.info(`Created batch job: ${batchJob.id}`);
|
|
return batchJob;
|
|
}
|
|
async monitorBatchJob(batchId) {
|
|
const pollInterval = 60;
|
|
let attempts = 0;
|
|
const maxAttempts = 120;
|
|
const startTime = Date.now();
|
|
let lastStatus = '';
|
|
while (attempts < maxAttempts) {
|
|
const batchJob = await this.client.batches.retrieve(batchId);
|
|
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
const statusSymbol = batchJob.status === 'in_progress' ? '⚙️' :
|
|
batchJob.status === 'finalizing' ? '📦' :
|
|
batchJob.status === 'validating' ? '🔍' :
|
|
batchJob.status === 'completed' ? '✅' :
|
|
batchJob.status === 'failed' ? '❌' : '⏳';
|
|
console.log(` ${statusSymbol} Batch ${batchId.slice(-8)}: ${batchJob.status} (${elapsedMinutes} min, check ${attempts + 1})`);
|
|
if (batchJob.status !== lastStatus) {
|
|
logger_1.logger.info(`Batch ${batchId} status changed: ${lastStatus} -> ${batchJob.status}`);
|
|
lastStatus = batchJob.status;
|
|
}
|
|
if (batchJob.status === 'completed') {
|
|
console.log(` ✅ Batch ${batchId.slice(-8)} completed successfully in ${elapsedMinutes} minutes`);
|
|
logger_1.logger.info(`Batch job ${batchId} completed successfully`);
|
|
return batchJob;
|
|
}
|
|
if (['failed', 'expired', 'cancelled'].includes(batchJob.status)) {
|
|
logger_1.logger.error(`Batch job ${batchId} failed with status: ${batchJob.status}`);
|
|
throw new Error(`Batch job failed with status: ${batchJob.status}`);
|
|
}
|
|
logger_1.logger.debug(`Waiting ${pollInterval} seconds before next check...`);
|
|
await this.sleep(pollInterval * 1000);
|
|
attempts++;
|
|
}
|
|
throw new Error(`Batch job monitoring timed out after ${maxAttempts} minutes`);
|
|
}
|
|
async retrieveResults(batchJob) {
|
|
const results = [];
|
|
if (batchJob.output_file_id) {
|
|
const fileResponse = await this.client.files.content(batchJob.output_file_id);
|
|
const fileContent = await fileResponse.text();
|
|
const lines = fileContent.trim().split('\n');
|
|
for (const line of lines) {
|
|
if (!line)
|
|
continue;
|
|
try {
|
|
const result = JSON.parse(line);
|
|
const parsed = this.generator.parseResult(result);
|
|
results.push(parsed);
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.error('Error parsing result line:', error);
|
|
}
|
|
}
|
|
logger_1.logger.info(`Retrieved ${results.length} successful results from batch job`);
|
|
}
|
|
if (batchJob.error_file_id) {
|
|
logger_1.logger.warn(`Batch job has error file: ${batchJob.error_file_id}`);
|
|
try {
|
|
const errorResponse = await this.client.files.content(batchJob.error_file_id);
|
|
const errorContent = await errorResponse.text();
|
|
const errorFilePath = path.join(this.outputDir, `batch_${batchJob.id}_error.jsonl`);
|
|
fs.writeFileSync(errorFilePath, errorContent);
|
|
logger_1.logger.warn(`Error file saved to: ${errorFilePath}`);
|
|
const errorLines = errorContent.trim().split('\n');
|
|
logger_1.logger.warn(`Found ${errorLines.length} failed requests in error file`);
|
|
for (const line of errorLines) {
|
|
if (!line)
|
|
continue;
|
|
try {
|
|
const errorResult = JSON.parse(line);
|
|
const templateId = parseInt(errorResult.custom_id?.replace('template-', '') || '0');
|
|
if (templateId > 0) {
|
|
const errorMessage = errorResult.response?.body?.error?.message ||
|
|
errorResult.error?.message ||
|
|
'Unknown error';
|
|
logger_1.logger.debug(`Template ${templateId} failed: ${errorMessage}`);
|
|
const defaultMeta = this.generator.getDefaultMetadata();
|
|
results.push({
|
|
templateId,
|
|
metadata: defaultMeta,
|
|
error: errorMessage
|
|
});
|
|
}
|
|
}
|
|
catch (parseError) {
|
|
logger_1.logger.error('Error parsing error line:', parseError);
|
|
}
|
|
}
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.error('Failed to process error file:', error);
|
|
}
|
|
}
|
|
if (results.length === 0 && !batchJob.output_file_id && !batchJob.error_file_id) {
|
|
throw new Error('No output file or error file available for batch job');
|
|
}
|
|
logger_1.logger.info(`Total results (successful + failed): ${results.length}`);
|
|
return results;
|
|
}
|
|
async cleanup(localFile, inputFileId, outputFileId) {
|
|
try {
|
|
fs.unlinkSync(localFile);
|
|
logger_1.logger.debug(`Deleted local file: ${localFile}`);
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.warn(`Failed to delete local file: ${localFile}`, error);
|
|
}
|
|
try {
|
|
await this.client.files.del(inputFileId);
|
|
logger_1.logger.debug(`Deleted input file from OpenAI: ${inputFileId}`);
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.warn(`Failed to delete input file from OpenAI: ${inputFileId}`, error);
|
|
}
|
|
if (outputFileId) {
|
|
try {
|
|
await this.client.files.del(outputFileId);
|
|
logger_1.logger.debug(`Deleted output file from OpenAI: ${outputFileId}`);
|
|
}
|
|
catch (error) {
|
|
logger_1.logger.warn(`Failed to delete output file from OpenAI: ${outputFileId}`, error);
|
|
}
|
|
}
|
|
}
|
|
sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
}
|
|
exports.BatchProcessor = BatchProcessor;
|
|
//# sourceMappingURL=batch-processor.js.map
|