const fs = require("fs-extra"); const path = require("node:path"); const os = require("node:os"); const { isBinaryFile } = require("./binary.js"); /** * Aggregate file contents with bounded concurrency. * Returns text files, binary files (with size), and errors. * @param {string[]} files absolute file paths * @param {string} rootDir * @param {{ text?: string, warn?: (msg: string) => void } | null} spinner */ async function aggregateFileContents(files, rootDir, spinner = null) { const results = { textFiles: [], binaryFiles: [], errors: [], totalFiles: files.length, processedFiles: 0, }; // Automatic concurrency selection based on CPU count and workload size. // - Base on 2x logical CPUs, clamped to [2, 64] // - For very small workloads, avoid excessive parallelism const cpuCount = (os.cpus && Array.isArray(os.cpus()) ? os.cpus().length : (os.cpus?.length || 4)); let concurrency = Math.min(64, Math.max(2, (Number(cpuCount) || 4) * 2)); if (files.length > 0 && files.length < concurrency) { concurrency = Math.max(1, Math.min(concurrency, Math.ceil(files.length / 2))); } async function processOne(filePath) { try { const relativePath = path.relative(rootDir, filePath); if (spinner) { spinner.text = `Processing: ${relativePath} (${results.processedFiles + 1}/${results.totalFiles})`; } const binary = await isBinaryFile(filePath); if (binary) { const size = (await fs.stat(filePath)).size; results.binaryFiles.push({ path: relativePath, absolutePath: filePath, size }); } else { const content = await fs.readFile(filePath, "utf8"); results.textFiles.push({ path: relativePath, absolutePath: filePath, content, size: content.length, lines: content.split("\n").length, }); } } catch (error) { const relativePath = path.relative(rootDir, filePath); const errorInfo = { path: relativePath, absolutePath: filePath, error: error.message }; results.errors.push(errorInfo); if (spinner) { spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`); } else { console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`); } } finally { results.processedFiles++; } } for (let i = 0; i < files.length; i += concurrency) { const slice = files.slice(i, i + concurrency); await Promise.all(slice.map(processOne)); } return results; } module.exports = { aggregateFileContents, };