77 lines
2.5 KiB
JavaScript
77 lines
2.5 KiB
JavaScript
const fs = require("fs-extra");
|
|
const path = require("node:path");
|
|
const os = require("node:os");
|
|
const { isBinaryFile } = require("./binary.js");
|
|
|
|
/**
|
|
* Aggregate file contents with bounded concurrency.
|
|
* Returns text files, binary files (with size), and errors.
|
|
* @param {string[]} files absolute file paths
|
|
* @param {string} rootDir
|
|
* @param {{ text?: string, warn?: (msg: string) => void } | null} spinner
|
|
*/
|
|
async function aggregateFileContents(files, rootDir, spinner = null) {
|
|
const results = {
|
|
textFiles: [],
|
|
binaryFiles: [],
|
|
errors: [],
|
|
totalFiles: files.length,
|
|
processedFiles: 0,
|
|
};
|
|
|
|
// Automatic concurrency selection based on CPU count and workload size.
|
|
// - Base on 2x logical CPUs, clamped to [2, 64]
|
|
// - For very small workloads, avoid excessive parallelism
|
|
const cpuCount = (os.cpus && Array.isArray(os.cpus()) ? os.cpus().length : (os.cpus?.length || 4));
|
|
let concurrency = Math.min(64, Math.max(2, (Number(cpuCount) || 4) * 2));
|
|
if (files.length > 0 && files.length < concurrency) {
|
|
concurrency = Math.max(1, Math.min(concurrency, Math.ceil(files.length / 2)));
|
|
}
|
|
|
|
async function processOne(filePath) {
|
|
try {
|
|
const relativePath = path.relative(rootDir, filePath);
|
|
if (spinner) {
|
|
spinner.text = `Processing: ${relativePath} (${results.processedFiles + 1}/${results.totalFiles})`;
|
|
}
|
|
|
|
const binary = await isBinaryFile(filePath);
|
|
if (binary) {
|
|
const size = (await fs.stat(filePath)).size;
|
|
results.binaryFiles.push({ path: relativePath, absolutePath: filePath, size });
|
|
} else {
|
|
const content = await fs.readFile(filePath, "utf8");
|
|
results.textFiles.push({
|
|
path: relativePath,
|
|
absolutePath: filePath,
|
|
content,
|
|
size: content.length,
|
|
lines: content.split("\n").length,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
const relativePath = path.relative(rootDir, filePath);
|
|
const errorInfo = { path: relativePath, absolutePath: filePath, error: error.message };
|
|
results.errors.push(errorInfo);
|
|
if (spinner) {
|
|
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
|
} else {
|
|
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
|
}
|
|
} finally {
|
|
results.processedFiles++;
|
|
}
|
|
}
|
|
|
|
for (let i = 0; i < files.length; i += concurrency) {
|
|
const slice = files.slice(i, i + concurrency);
|
|
await Promise.all(slice.map(processOne));
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
module.exports = {
|
|
aggregateFileContents,
|
|
};
|