feat: modularize flattener tool into separate components with improved project root detection (#417)
This commit is contained in:
manjaroblack
4
.gitignore
vendored
4
.gitignore
vendored
@@ -3,6 +3,8 @@ node_modules/
|
|||||||
pnpm-lock.yaml
|
pnpm-lock.yaml
|
||||||
bun.lock
|
bun.lock
|
||||||
deno.lock
|
deno.lock
|
||||||
|
pnpm-workspace.yaml
|
||||||
|
package-lock.json
|
||||||
|
|
||||||
# Logs
|
# Logs
|
||||||
logs/
|
logs/
|
||||||
@@ -41,3 +43,5 @@ CLAUDE.md
|
|||||||
.bmad-creator-tools
|
.bmad-creator-tools
|
||||||
test-project-install/*
|
test-project-install/*
|
||||||
sample-project/*
|
sample-project/*
|
||||||
|
flattened-codebase.xml
|
||||||
|
|
||||||
|
|||||||
37
README.md
37
README.md
@@ -144,7 +144,7 @@ npx bmad-method flatten --input /path/to/source --output /path/to/output/codebas
|
|||||||
|
|
||||||
The tool will display progress and provide a comprehensive summary:
|
The tool will display progress and provide a comprehensive summary:
|
||||||
|
|
||||||
```
|
```text
|
||||||
📊 Completion Summary:
|
📊 Completion Summary:
|
||||||
✅ Successfully processed 156 files into flattened-codebase.xml
|
✅ Successfully processed 156 files into flattened-codebase.xml
|
||||||
📁 Output file: /path/to/your/project/flattened-codebase.xml
|
📁 Output file: /path/to/your/project/flattened-codebase.xml
|
||||||
@@ -155,7 +155,40 @@ The tool will display progress and provide a comprehensive summary:
|
|||||||
📊 File breakdown: 142 text, 14 binary, 0 errors
|
📊 File breakdown: 142 text, 14 binary, 0 errors
|
||||||
```
|
```
|
||||||
|
|
||||||
The generated XML file contains all your project's source code in a structured format that AI models can easily parse and understand, making it perfect for code reviews, architecture discussions, or getting AI assistance with your BMad-Method projects.
|
The generated XML file contains your project's text-based source files in a structured format that AI models can easily parse and understand, making it perfect for code reviews, architecture discussions, or getting AI assistance with your BMad-Method projects.
|
||||||
|
|
||||||
|
#### Advanced Usage & Options
|
||||||
|
|
||||||
|
- CLI options
|
||||||
|
- `-i, --input <path>`: Directory to flatten. Default: current working directory or auto-detected project root when run interactively.
|
||||||
|
- `-o, --output <path>`: Output file path. Default: `flattened-codebase.xml` in the chosen directory.
|
||||||
|
- Interactive mode
|
||||||
|
- If you do not pass `--input` and `--output` and the terminal is interactive (TTY), the tool will attempt to detect your project root (by looking for markers like `.git`, `package.json`, etc.) and prompt you to confirm or override the paths.
|
||||||
|
- In non-interactive contexts (e.g., CI), it will prefer the detected root silently; otherwise it falls back to the current directory and default filename.
|
||||||
|
- File discovery and ignoring
|
||||||
|
- Uses `git ls-files` when inside a git repository for speed and correctness; otherwise falls back to a glob-based scan.
|
||||||
|
- Applies your `.gitignore` plus a curated set of default ignore patterns (e.g., `node_modules`, build outputs, caches, logs, IDE folders, lockfiles, large media/binaries, `.env*`, and previously generated XML outputs).
|
||||||
|
- Binary handling
|
||||||
|
- Binary files are detected and excluded from the XML content. They are counted in the final summary but not embedded in the output.
|
||||||
|
- XML format and safety
|
||||||
|
- UTF-8 encoded file with root element `<files>`.
|
||||||
|
- Each text file is emitted as a `<file path="relative/path">` element whose content is wrapped in `<![CDATA[ ... ]]>`.
|
||||||
|
- The tool safely handles occurrences of `]]>` inside content by splitting the CDATA to preserve correctness.
|
||||||
|
- File contents are preserved as-is and indented for readability inside the XML.
|
||||||
|
- Performance
|
||||||
|
- Concurrency is selected automatically based on your CPU and workload size. No configuration required.
|
||||||
|
- Running inside a git repo improves discovery performance.
|
||||||
|
|
||||||
|
#### Minimal XML example
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<files>
|
||||||
|
<file path="src/index.js"><![CDATA[
|
||||||
|
// your source content
|
||||||
|
]]></file>
|
||||||
|
</files>
|
||||||
|
```
|
||||||
|
|
||||||
## Documentation & Resources
|
## Documentation & Resources
|
||||||
|
|
||||||
|
|||||||
78
package-lock.json
generated
78
package-lock.json
generated
@@ -15,9 +15,9 @@
|
|||||||
"commander": "^14.0.0",
|
"commander": "^14.0.0",
|
||||||
"fs-extra": "^11.3.0",
|
"fs-extra": "^11.3.0",
|
||||||
"glob": "^11.0.3",
|
"glob": "^11.0.3",
|
||||||
|
"ignore": "^7.0.5",
|
||||||
"inquirer": "^8.2.6",
|
"inquirer": "^8.2.6",
|
||||||
"js-yaml": "^4.1.0",
|
"js-yaml": "^4.1.0",
|
||||||
"minimatch": "^10.0.3",
|
|
||||||
"ora": "^5.4.1"
|
"ora": "^5.4.1"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
@@ -593,6 +593,22 @@
|
|||||||
"tslib": "^2.4.0"
|
"tslib": "^2.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@inquirer/external-editor": {
|
||||||
|
"version": "1.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.0.tgz",
|
||||||
|
"integrity": "sha512-5v3YXc5ZMfL6OJqXPrX9csb4l7NlQA2doO1yynUjpUChT9hg4JcuBVP0RbsEJ/3SL/sxWEyFjT2W69ZhtoBWqg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"chardet": "^2.1.0",
|
||||||
|
"iconv-lite": "^0.6.3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@types/node": ">=18"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@isaacs/balanced-match": {
|
"node_modules/@isaacs/balanced-match": {
|
||||||
"version": "4.0.1",
|
"version": "4.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
|
||||||
@@ -2184,7 +2200,6 @@
|
|||||||
"version": "24.1.0",
|
"version": "24.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz",
|
||||||
"integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==",
|
"integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~7.8.0"
|
"undici-types": "~7.8.0"
|
||||||
@@ -3004,9 +3019,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/chardet": {
|
"node_modules/chardet": {
|
||||||
"version": "0.7.0",
|
"version": "2.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
|
"resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.0.tgz",
|
||||||
"integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
|
"integrity": "sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/ci-info": {
|
"node_modules/ci-info": {
|
||||||
@@ -3971,20 +3986,6 @@
|
|||||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/external-editor": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==",
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"chardet": "^0.7.0",
|
|
||||||
"iconv-lite": "^0.4.24",
|
|
||||||
"tmp": "^0.0.33"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/fast-glob": {
|
"node_modules/fast-glob": {
|
||||||
"version": "3.3.3",
|
"version": "3.3.3",
|
||||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||||
@@ -4504,12 +4505,12 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/iconv-lite": {
|
"node_modules/iconv-lite": {
|
||||||
"version": "0.4.24",
|
"version": "0.6.3",
|
||||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"safer-buffer": ">= 2.1.2 < 3"
|
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=0.10.0"
|
"node": ">=0.10.0"
|
||||||
@@ -4539,7 +4540,6 @@
|
|||||||
"version": "7.0.5",
|
"version": "7.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz",
|
||||||
"integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==",
|
"integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 4"
|
"node": ">= 4"
|
||||||
@@ -4676,16 +4676,16 @@
|
|||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/inquirer": {
|
"node_modules/inquirer": {
|
||||||
"version": "8.2.6",
|
"version": "8.2.7",
|
||||||
"resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.6.tgz",
|
"resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz",
|
||||||
"integrity": "sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==",
|
"integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@inquirer/external-editor": "^1.0.0",
|
||||||
"ansi-escapes": "^4.2.1",
|
"ansi-escapes": "^4.2.1",
|
||||||
"chalk": "^4.1.1",
|
"chalk": "^4.1.1",
|
||||||
"cli-cursor": "^3.1.0",
|
"cli-cursor": "^3.1.0",
|
||||||
"cli-width": "^3.0.0",
|
"cli-width": "^3.0.0",
|
||||||
"external-editor": "^3.0.3",
|
|
||||||
"figures": "^3.0.0",
|
"figures": "^3.0.0",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"mute-stream": "0.0.8",
|
"mute-stream": "0.0.8",
|
||||||
@@ -10100,15 +10100,6 @@
|
|||||||
"url": "https://github.com/sponsors/sindresorhus"
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/os-tmpdir": {
|
|
||||||
"version": "1.0.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
|
|
||||||
"integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=0.10.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/p-each-series": {
|
"node_modules/p-each-series": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-3.0.0.tgz",
|
||||||
@@ -11963,18 +11954,6 @@
|
|||||||
"safe-buffer": "~5.1.0"
|
"safe-buffer": "~5.1.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/tmp": {
|
|
||||||
"version": "0.0.33",
|
|
||||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
|
|
||||||
"integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"os-tmpdir": "~1.0.2"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=0.6.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/tmpl": {
|
"node_modules/tmpl": {
|
||||||
"version": "1.0.5",
|
"version": "1.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz",
|
||||||
@@ -12064,7 +12043,6 @@
|
|||||||
"version": "7.8.0",
|
"version": "7.8.0",
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",
|
||||||
"integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==",
|
"integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/unicode-emoji-modifier-base": {
|
"node_modules/unicode-emoji-modifier-base": {
|
||||||
|
|||||||
@@ -40,9 +40,9 @@
|
|||||||
"commander": "^14.0.0",
|
"commander": "^14.0.0",
|
||||||
"fs-extra": "^11.3.0",
|
"fs-extra": "^11.3.0",
|
||||||
"glob": "^11.0.3",
|
"glob": "^11.0.3",
|
||||||
|
"ignore": "^7.0.5",
|
||||||
"inquirer": "^8.2.6",
|
"inquirer": "^8.2.6",
|
||||||
"js-yaml": "^4.1.0",
|
"js-yaml": "^4.1.0",
|
||||||
"minimatch": "^10.0.3",
|
|
||||||
"ora": "^5.4.1"
|
"ora": "^5.4.1"
|
||||||
},
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
|
|||||||
76
tools/flattener/aggregate.js
Normal file
76
tools/flattener/aggregate.js
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
const fs = require("fs-extra");
|
||||||
|
const path = require("node:path");
|
||||||
|
const os = require("node:os");
|
||||||
|
const { isBinaryFile } = require("./binary.js");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Aggregate file contents with bounded concurrency.
|
||||||
|
* Returns text files, binary files (with size), and errors.
|
||||||
|
* @param {string[]} files absolute file paths
|
||||||
|
* @param {string} rootDir
|
||||||
|
* @param {{ text?: string, warn?: (msg: string) => void } | null} spinner
|
||||||
|
*/
|
||||||
|
async function aggregateFileContents(files, rootDir, spinner = null) {
|
||||||
|
const results = {
|
||||||
|
textFiles: [],
|
||||||
|
binaryFiles: [],
|
||||||
|
errors: [],
|
||||||
|
totalFiles: files.length,
|
||||||
|
processedFiles: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Automatic concurrency selection based on CPU count and workload size.
|
||||||
|
// - Base on 2x logical CPUs, clamped to [2, 64]
|
||||||
|
// - For very small workloads, avoid excessive parallelism
|
||||||
|
const cpuCount = (os.cpus && Array.isArray(os.cpus()) ? os.cpus().length : (os.cpus?.length || 4));
|
||||||
|
let concurrency = Math.min(64, Math.max(2, (Number(cpuCount) || 4) * 2));
|
||||||
|
if (files.length > 0 && files.length < concurrency) {
|
||||||
|
concurrency = Math.max(1, Math.min(concurrency, Math.ceil(files.length / 2)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processOne(filePath) {
|
||||||
|
try {
|
||||||
|
const relativePath = path.relative(rootDir, filePath);
|
||||||
|
if (spinner) {
|
||||||
|
spinner.text = `Processing: ${relativePath} (${results.processedFiles + 1}/${results.totalFiles})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const binary = await isBinaryFile(filePath);
|
||||||
|
if (binary) {
|
||||||
|
const size = (await fs.stat(filePath)).size;
|
||||||
|
results.binaryFiles.push({ path: relativePath, absolutePath: filePath, size });
|
||||||
|
} else {
|
||||||
|
const content = await fs.readFile(filePath, "utf8");
|
||||||
|
results.textFiles.push({
|
||||||
|
path: relativePath,
|
||||||
|
absolutePath: filePath,
|
||||||
|
content,
|
||||||
|
size: content.length,
|
||||||
|
lines: content.split("\n").length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const relativePath = path.relative(rootDir, filePath);
|
||||||
|
const errorInfo = { path: relativePath, absolutePath: filePath, error: error.message };
|
||||||
|
results.errors.push(errorInfo);
|
||||||
|
if (spinner) {
|
||||||
|
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||||
|
} else {
|
||||||
|
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
results.processedFiles++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < files.length; i += concurrency) {
|
||||||
|
const slice = files.slice(i, i + concurrency);
|
||||||
|
await Promise.all(slice.map(processOne));
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
aggregateFileContents,
|
||||||
|
};
|
||||||
53
tools/flattener/binary.js
Normal file
53
tools/flattener/binary.js
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
const fsp = require("node:fs/promises");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { Buffer } = require("node:buffer");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Efficiently determine if a file is binary without reading the whole file.
|
||||||
|
* - Fast path by extension for common binaries
|
||||||
|
* - Otherwise read a small prefix and check for NUL bytes
|
||||||
|
* @param {string} filePath
|
||||||
|
* @returns {Promise<boolean>}
|
||||||
|
*/
|
||||||
|
async function isBinaryFile(filePath) {
|
||||||
|
try {
|
||||||
|
const stats = await fsp.stat(filePath);
|
||||||
|
if (stats.isDirectory()) {
|
||||||
|
throw new Error("EISDIR: illegal operation on a directory");
|
||||||
|
}
|
||||||
|
|
||||||
|
const binaryExtensions = new Set([
|
||||||
|
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".ico", ".svg",
|
||||||
|
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||||
|
".zip", ".tar", ".gz", ".rar", ".7z",
|
||||||
|
".exe", ".dll", ".so", ".dylib",
|
||||||
|
".mp3", ".mp4", ".avi", ".mov", ".wav",
|
||||||
|
".ttf", ".otf", ".woff", ".woff2",
|
||||||
|
".bin", ".dat", ".db", ".sqlite",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const ext = path.extname(filePath).toLowerCase();
|
||||||
|
if (binaryExtensions.has(ext)) return true;
|
||||||
|
if (stats.size === 0) return false;
|
||||||
|
|
||||||
|
const sampleSize = Math.min(4096, stats.size);
|
||||||
|
const fd = await fsp.open(filePath, "r");
|
||||||
|
try {
|
||||||
|
const buffer = Buffer.allocUnsafe(sampleSize);
|
||||||
|
const { bytesRead } = await fd.read(buffer, 0, sampleSize, 0);
|
||||||
|
const slice = bytesRead === sampleSize ? buffer : buffer.subarray(0, bytesRead);
|
||||||
|
return slice.includes(0);
|
||||||
|
} finally {
|
||||||
|
await fd.close();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(
|
||||||
|
`Warning: Could not determine if file is binary: ${filePath} - ${error.message}`,
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
isBinaryFile,
|
||||||
|
};
|
||||||
70
tools/flattener/discovery.js
Normal file
70
tools/flattener/discovery.js
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
const path = require("node:path");
|
||||||
|
const { execFile } = require("node:child_process");
|
||||||
|
const { promisify } = require("node:util");
|
||||||
|
const { glob } = require("glob");
|
||||||
|
const { loadIgnore } = require("./ignoreRules.js");
|
||||||
|
|
||||||
|
const pExecFile = promisify(execFile);
|
||||||
|
|
||||||
|
async function isGitRepo(rootDir) {
|
||||||
|
try {
|
||||||
|
const { stdout } = await pExecFile("git", [
|
||||||
|
"rev-parse",
|
||||||
|
"--is-inside-work-tree",
|
||||||
|
], { cwd: rootDir });
|
||||||
|
return String(stdout || "").toString().trim() === "true";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gitListFiles(rootDir) {
|
||||||
|
try {
|
||||||
|
const { stdout } = await pExecFile("git", [
|
||||||
|
"ls-files",
|
||||||
|
"-co",
|
||||||
|
"--exclude-standard",
|
||||||
|
], { cwd: rootDir });
|
||||||
|
return String(stdout || "")
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Discover files under rootDir.
|
||||||
|
* - Prefer git ls-files when available for speed/correctness
|
||||||
|
* - Fallback to glob and apply unified ignore rules
|
||||||
|
* @param {string} rootDir
|
||||||
|
* @param {object} [options]
|
||||||
|
* @param {boolean} [options.preferGit=true]
|
||||||
|
* @returns {Promise<string[]>} absolute file paths
|
||||||
|
*/
|
||||||
|
async function discoverFiles(rootDir, options = {}) {
|
||||||
|
const { preferGit = true } = options;
|
||||||
|
const { filter } = await loadIgnore(rootDir);
|
||||||
|
|
||||||
|
// Try git first
|
||||||
|
if (preferGit && await isGitRepo(rootDir)) {
|
||||||
|
const relFiles = await gitListFiles(rootDir);
|
||||||
|
const filteredRel = relFiles.filter((p) => filter(p));
|
||||||
|
return filteredRel.map((p) => path.resolve(rootDir, p));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Glob fallback
|
||||||
|
const globbed = await glob("**/*", {
|
||||||
|
cwd: rootDir,
|
||||||
|
nodir: true,
|
||||||
|
dot: true,
|
||||||
|
follow: false,
|
||||||
|
});
|
||||||
|
const filteredRel = globbed.filter((p) => filter(p));
|
||||||
|
return filteredRel.map((p) => path.resolve(rootDir, p));
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
discoverFiles,
|
||||||
|
};
|
||||||
35
tools/flattener/files.js
Normal file
35
tools/flattener/files.js
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
const path = require("node:path");
|
||||||
|
const discovery = require("./discovery.js");
|
||||||
|
const ignoreRules = require("./ignoreRules.js");
|
||||||
|
const { isBinaryFile } = require("./binary.js");
|
||||||
|
const { aggregateFileContents } = require("./aggregate.js");
|
||||||
|
|
||||||
|
// Backward-compatible signature; delegate to central loader
|
||||||
|
async function parseGitignore(gitignorePath) {
|
||||||
|
return await ignoreRules.parseGitignore(gitignorePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverFiles(rootDir) {
|
||||||
|
try {
|
||||||
|
// Delegate to discovery module which respects .gitignore and defaults
|
||||||
|
return await discovery.discoverFiles(rootDir, { preferGit: true });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error discovering files:", error.message);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function filterFiles(files, rootDir) {
|
||||||
|
const { filter } = await ignoreRules.loadIgnore(rootDir);
|
||||||
|
const relativeFiles = files.map((f) => path.relative(rootDir, f));
|
||||||
|
const filteredRelative = relativeFiles.filter((p) => filter(p));
|
||||||
|
return filteredRelative.map((p) => path.resolve(rootDir, p));
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
parseGitignore,
|
||||||
|
discoverFiles,
|
||||||
|
isBinaryFile,
|
||||||
|
aggregateFileContents,
|
||||||
|
filterFiles,
|
||||||
|
};
|
||||||
176
tools/flattener/ignoreRules.js
Normal file
176
tools/flattener/ignoreRules.js
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
const fs = require("fs-extra");
|
||||||
|
const path = require("node:path");
|
||||||
|
const ignore = require("ignore");
|
||||||
|
|
||||||
|
// Central default ignore patterns for discovery and filtering.
|
||||||
|
// These complement .gitignore and are applied regardless of VCS presence.
|
||||||
|
const DEFAULT_PATTERNS = [
|
||||||
|
// Project/VCS
|
||||||
|
"**/.bmad-core/**",
|
||||||
|
"**/.git/**",
|
||||||
|
"**/.svn/**",
|
||||||
|
"**/.hg/**",
|
||||||
|
"**/.bzr/**",
|
||||||
|
// Package/build outputs
|
||||||
|
"**/node_modules/**",
|
||||||
|
"**/bower_components/**",
|
||||||
|
"**/vendor/**",
|
||||||
|
"**/packages/**",
|
||||||
|
"**/build/**",
|
||||||
|
"**/dist/**",
|
||||||
|
"**/out/**",
|
||||||
|
"**/target/**",
|
||||||
|
"**/bin/**",
|
||||||
|
"**/obj/**",
|
||||||
|
"**/release/**",
|
||||||
|
"**/debug/**",
|
||||||
|
// Environments
|
||||||
|
"**/.venv/**",
|
||||||
|
"**/venv/**",
|
||||||
|
"**/.virtualenv/**",
|
||||||
|
"**/virtualenv/**",
|
||||||
|
"**/env/**",
|
||||||
|
// Logs & coverage
|
||||||
|
"**/*.log",
|
||||||
|
"**/npm-debug.log*",
|
||||||
|
"**/yarn-debug.log*",
|
||||||
|
"**/yarn-error.log*",
|
||||||
|
"**/lerna-debug.log*",
|
||||||
|
"**/coverage/**",
|
||||||
|
"**/.nyc_output/**",
|
||||||
|
"**/.coverage/**",
|
||||||
|
"**/test-results/**",
|
||||||
|
// Caches & temp
|
||||||
|
"**/.cache/**",
|
||||||
|
"**/.tmp/**",
|
||||||
|
"**/.temp/**",
|
||||||
|
"**/tmp/**",
|
||||||
|
"**/temp/**",
|
||||||
|
"**/.sass-cache/**",
|
||||||
|
// IDE/editor
|
||||||
|
"**/.vscode/**",
|
||||||
|
"**/.idea/**",
|
||||||
|
"**/*.swp",
|
||||||
|
"**/*.swo",
|
||||||
|
"**/*~",
|
||||||
|
"**/.project",
|
||||||
|
"**/.classpath",
|
||||||
|
"**/.settings/**",
|
||||||
|
"**/*.sublime-project",
|
||||||
|
"**/*.sublime-workspace",
|
||||||
|
// Lockfiles
|
||||||
|
"**/package-lock.json",
|
||||||
|
"**/yarn.lock",
|
||||||
|
"**/pnpm-lock.yaml",
|
||||||
|
"**/composer.lock",
|
||||||
|
"**/Pipfile.lock",
|
||||||
|
// Python/Java/compiled artifacts
|
||||||
|
"**/*.pyc",
|
||||||
|
"**/*.pyo",
|
||||||
|
"**/*.pyd",
|
||||||
|
"**/__pycache__/**",
|
||||||
|
"**/*.class",
|
||||||
|
"**/*.jar",
|
||||||
|
"**/*.war",
|
||||||
|
"**/*.ear",
|
||||||
|
"**/*.o",
|
||||||
|
"**/*.so",
|
||||||
|
"**/*.dll",
|
||||||
|
"**/*.exe",
|
||||||
|
// System junk
|
||||||
|
"**/lib64/**",
|
||||||
|
"**/.venv/lib64/**",
|
||||||
|
"**/venv/lib64/**",
|
||||||
|
"**/_site/**",
|
||||||
|
"**/.jekyll-cache/**",
|
||||||
|
"**/.jekyll-metadata",
|
||||||
|
"**/.DS_Store",
|
||||||
|
"**/.DS_Store?",
|
||||||
|
"**/._*",
|
||||||
|
"**/.Spotlight-V100/**",
|
||||||
|
"**/.Trashes/**",
|
||||||
|
"**/ehthumbs.db",
|
||||||
|
"**/Thumbs.db",
|
||||||
|
"**/desktop.ini",
|
||||||
|
// XML outputs
|
||||||
|
"**/flattened-codebase.xml",
|
||||||
|
"**/repomix-output.xml",
|
||||||
|
// Images, media, fonts, archives, docs, dylibs
|
||||||
|
"**/*.jpg",
|
||||||
|
"**/*.jpeg",
|
||||||
|
"**/*.png",
|
||||||
|
"**/*.gif",
|
||||||
|
"**/*.bmp",
|
||||||
|
"**/*.ico",
|
||||||
|
"**/*.svg",
|
||||||
|
"**/*.pdf",
|
||||||
|
"**/*.doc",
|
||||||
|
"**/*.docx",
|
||||||
|
"**/*.xls",
|
||||||
|
"**/*.xlsx",
|
||||||
|
"**/*.ppt",
|
||||||
|
"**/*.pptx",
|
||||||
|
"**/*.zip",
|
||||||
|
"**/*.tar",
|
||||||
|
"**/*.gz",
|
||||||
|
"**/*.rar",
|
||||||
|
"**/*.7z",
|
||||||
|
"**/*.dylib",
|
||||||
|
"**/*.mp3",
|
||||||
|
"**/*.mp4",
|
||||||
|
"**/*.avi",
|
||||||
|
"**/*.mov",
|
||||||
|
"**/*.wav",
|
||||||
|
"**/*.ttf",
|
||||||
|
"**/*.otf",
|
||||||
|
"**/*.woff",
|
||||||
|
"**/*.woff2",
|
||||||
|
// Env files
|
||||||
|
"**/.env",
|
||||||
|
"**/.env.*",
|
||||||
|
"**/*.env",
|
||||||
|
// Misc
|
||||||
|
"**/junit.xml",
|
||||||
|
];
|
||||||
|
|
||||||
|
async function readIgnoreFile(filePath) {
|
||||||
|
try {
|
||||||
|
if (!await fs.pathExists(filePath)) return [];
|
||||||
|
const content = await fs.readFile(filePath, "utf8");
|
||||||
|
return content
|
||||||
|
.split("\n")
|
||||||
|
.map((l) => l.trim())
|
||||||
|
.filter((l) => l && !l.startsWith("#"));
|
||||||
|
} catch (err) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backward compatible export matching previous signature
|
||||||
|
async function parseGitignore(gitignorePath) {
|
||||||
|
return readIgnoreFile(gitignorePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadIgnore(rootDir, extraPatterns = []) {
|
||||||
|
const ig = ignore();
|
||||||
|
const gitignorePath = path.join(rootDir, ".gitignore");
|
||||||
|
const patterns = [
|
||||||
|
...await readIgnoreFile(gitignorePath),
|
||||||
|
...DEFAULT_PATTERNS,
|
||||||
|
...extraPatterns,
|
||||||
|
];
|
||||||
|
// De-duplicate
|
||||||
|
const unique = Array.from(new Set(patterns.map((p) => String(p))));
|
||||||
|
ig.add(unique);
|
||||||
|
|
||||||
|
// Include-only filter: return true if path should be included
|
||||||
|
const filter = (relativePath) => !ig.ignores(relativePath.replace(/\\/g, "/"));
|
||||||
|
|
||||||
|
return { ig, filter, patterns: unique };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
DEFAULT_PATTERNS,
|
||||||
|
parseGitignore,
|
||||||
|
loadIgnore,
|
||||||
|
};
|
||||||
@@ -1,258 +1,38 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
const { Command } = require('commander');
|
const { Command } = require("commander");
|
||||||
const fs = require('fs-extra');
|
const fs = require("fs-extra");
|
||||||
const path = require('node:path');
|
const path = require("node:path");
|
||||||
const { glob } = require('glob');
|
const process = require("node:process");
|
||||||
const { minimatch } = require('minimatch');
|
|
||||||
|
// Modularized components
|
||||||
|
const { findProjectRoot } = require("./projectRoot.js");
|
||||||
|
const { promptYesNo, promptPath } = require("./prompts.js");
|
||||||
|
const {
|
||||||
|
discoverFiles,
|
||||||
|
filterFiles,
|
||||||
|
aggregateFileContents,
|
||||||
|
} = require("./files.js");
|
||||||
|
const { generateXMLOutput } = require("./xml.js");
|
||||||
|
const { calculateStatistics } = require("./stats.js");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Recursively discover all files in a directory
|
* Recursively discover all files in a directory
|
||||||
* @param {string} rootDir - The root directory to scan
|
* @param {string} rootDir - The root directory to scan
|
||||||
* @returns {Promise<string[]>} Array of file paths
|
* @returns {Promise<string[]>} Array of file paths
|
||||||
*/
|
*/
|
||||||
async function discoverFiles(rootDir) {
|
|
||||||
try {
|
|
||||||
const gitignorePath = path.join(rootDir, '.gitignore');
|
|
||||||
const gitignorePatterns = await parseGitignore(gitignorePath);
|
|
||||||
|
|
||||||
// Common gitignore patterns that should always be ignored
|
|
||||||
const commonIgnorePatterns = [
|
|
||||||
// Version control
|
|
||||||
'.git/**',
|
|
||||||
'.svn/**',
|
|
||||||
'.hg/**',
|
|
||||||
'.bzr/**',
|
|
||||||
|
|
||||||
// Dependencies
|
|
||||||
'node_modules/**',
|
|
||||||
'bower_components/**',
|
|
||||||
'vendor/**',
|
|
||||||
'packages/**',
|
|
||||||
|
|
||||||
// Build outputs
|
|
||||||
'build/**',
|
|
||||||
'dist/**',
|
|
||||||
'out/**',
|
|
||||||
'target/**',
|
|
||||||
'bin/**',
|
|
||||||
'obj/**',
|
|
||||||
'release/**',
|
|
||||||
'debug/**',
|
|
||||||
|
|
||||||
// Environment and config
|
|
||||||
'.env',
|
|
||||||
'.env.*',
|
|
||||||
'*.env',
|
|
||||||
'.config',
|
|
||||||
'.venv/**',
|
|
||||||
'*/.venv/**',
|
|
||||||
'**/.venv/**',
|
|
||||||
'.venv',
|
|
||||||
'venv/**',
|
|
||||||
'*/venv/**',
|
|
||||||
'**/venv/**',
|
|
||||||
'venv',
|
|
||||||
'env/**',
|
|
||||||
'*/env/**',
|
|
||||||
'**/env/**',
|
|
||||||
'virtualenv/**',
|
|
||||||
'*/virtualenv/**',
|
|
||||||
'**/virtualenv/**',
|
|
||||||
|
|
||||||
// Logs
|
|
||||||
'logs/**',
|
|
||||||
'*.log',
|
|
||||||
'npm-debug.log*',
|
|
||||||
'yarn-debug.log*',
|
|
||||||
'yarn-error.log*',
|
|
||||||
'lerna-debug.log*',
|
|
||||||
|
|
||||||
// Coverage and testing
|
|
||||||
'coverage/**',
|
|
||||||
'.nyc_output/**',
|
|
||||||
'.coverage/**',
|
|
||||||
'test-results/**',
|
|
||||||
'junit.xml',
|
|
||||||
|
|
||||||
// Cache directories
|
|
||||||
'.cache/**',
|
|
||||||
'.tmp/**',
|
|
||||||
'.temp/**',
|
|
||||||
'tmp/**',
|
|
||||||
'temp/**',
|
|
||||||
'.sass-cache/**',
|
|
||||||
'.eslintcache',
|
|
||||||
'.stylelintcache',
|
|
||||||
|
|
||||||
// OS generated files
|
|
||||||
'.DS_Store',
|
|
||||||
'.DS_Store?',
|
|
||||||
'._*',
|
|
||||||
'.Spotlight-V100',
|
|
||||||
'.Trashes',
|
|
||||||
'ehthumbs.db',
|
|
||||||
'Thumbs.db',
|
|
||||||
'desktop.ini',
|
|
||||||
|
|
||||||
// IDE and editor files
|
|
||||||
'.vscode/**',
|
|
||||||
'.idea/**',
|
|
||||||
'*.swp',
|
|
||||||
'*.swo',
|
|
||||||
'*~',
|
|
||||||
'.project',
|
|
||||||
'.classpath',
|
|
||||||
'.settings/**',
|
|
||||||
'*.sublime-project',
|
|
||||||
'*.sublime-workspace',
|
|
||||||
|
|
||||||
// Package manager files
|
|
||||||
'package-lock.json',
|
|
||||||
'yarn.lock',
|
|
||||||
'pnpm-lock.yaml',
|
|
||||||
'composer.lock',
|
|
||||||
'Pipfile.lock',
|
|
||||||
|
|
||||||
// Runtime and compiled files
|
|
||||||
'*.pyc',
|
|
||||||
'*.pyo',
|
|
||||||
'*.pyd',
|
|
||||||
'__pycache__/**',
|
|
||||||
'*.class',
|
|
||||||
'*.jar',
|
|
||||||
'*.war',
|
|
||||||
'*.ear',
|
|
||||||
'*.o',
|
|
||||||
'*.so',
|
|
||||||
'*.dll',
|
|
||||||
'*.exe',
|
|
||||||
'lib64/**',
|
|
||||||
'**/.venv/lib64/**',
|
|
||||||
'**/venv/lib64/**',
|
|
||||||
|
|
||||||
// Documentation build
|
|
||||||
'_site/**',
|
|
||||||
'.jekyll-cache/**',
|
|
||||||
'.jekyll-metadata',
|
|
||||||
|
|
||||||
// Flattener specific outputs
|
|
||||||
'flattened-codebase.xml',
|
|
||||||
'repomix-output.xml'
|
|
||||||
];
|
|
||||||
|
|
||||||
const combinedIgnores = [
|
|
||||||
...gitignorePatterns,
|
|
||||||
...commonIgnorePatterns
|
|
||||||
];
|
|
||||||
|
|
||||||
// Add specific patterns for commonly ignored directories and files
|
|
||||||
const additionalGlobIgnores = [
|
|
||||||
// Virtual environments
|
|
||||||
'**/.venv/**', '**/venv/**', '**/.virtualenv/**', '**/virtualenv/**',
|
|
||||||
// Node modules
|
|
||||||
'**/node_modules/**',
|
|
||||||
// Python cache
|
|
||||||
'**/__pycache__/**', '**/*.pyc', '**/*.pyo', '**/*.pyd',
|
|
||||||
// Binary and media files
|
|
||||||
'**/*.jpg', '**/*.jpeg', '**/*.png', '**/*.gif', '**/*.bmp', '**/*.ico', '**/*.svg',
|
|
||||||
'**/*.pdf', '**/*.doc', '**/*.docx', '**/*.xls', '**/*.xlsx', '**/*.ppt', '**/*.pptx',
|
|
||||||
'**/*.zip', '**/*.tar', '**/*.gz', '**/*.rar', '**/*.7z',
|
|
||||||
'**/*.exe', '**/*.dll', '**/*.so', '**/*.dylib',
|
|
||||||
'**/*.mp3', '**/*.mp4', '**/*.avi', '**/*.mov', '**/*.wav',
|
|
||||||
'**/*.ttf', '**/*.otf', '**/*.woff', '**/*.woff2'
|
|
||||||
];
|
|
||||||
|
|
||||||
// Use glob to recursively find all files, excluding common ignore patterns
|
|
||||||
const files = await glob('**/*', {
|
|
||||||
cwd: rootDir,
|
|
||||||
nodir: true, // Only files, not directories
|
|
||||||
dot: true, // Include hidden files
|
|
||||||
follow: false, // Don't follow symbolic links
|
|
||||||
ignore: [...combinedIgnores, ...additionalGlobIgnores]
|
|
||||||
});
|
|
||||||
|
|
||||||
return files.map(file => path.resolve(rootDir, file));
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error discovering files:', error.message);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse .gitignore file and return ignore patterns
|
* Parse .gitignore file and return ignore patterns
|
||||||
* @param {string} gitignorePath - Path to .gitignore file
|
* @param {string} gitignorePath - Path to .gitignore file
|
||||||
* @returns {Promise<string[]>} Array of ignore patterns
|
* @returns {Promise<string[]>} Array of ignore patterns
|
||||||
*/
|
*/
|
||||||
async function parseGitignore(gitignorePath) {
|
|
||||||
try {
|
|
||||||
if (!await fs.pathExists(gitignorePath)) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const content = await fs.readFile(gitignorePath, 'utf8');
|
|
||||||
return content
|
|
||||||
.split('\n')
|
|
||||||
.map(line => line.trim())
|
|
||||||
.filter(line => line && !line.startsWith('#')) // Remove empty lines and comments
|
|
||||||
.map(pattern => {
|
|
||||||
// Convert gitignore patterns to glob patterns
|
|
||||||
if (pattern.endsWith('/')) {
|
|
||||||
return pattern + '**';
|
|
||||||
}
|
|
||||||
return pattern;
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error parsing .gitignore:', error.message);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a file is binary using file command and heuristics
|
* Check if a file is binary using file command and heuristics
|
||||||
* @param {string} filePath - Path to the file
|
* @param {string} filePath - Path to the file
|
||||||
* @returns {Promise<boolean>} True if file is binary
|
* @returns {Promise<boolean>} True if file is binary
|
||||||
*/
|
*/
|
||||||
async function isBinaryFile(filePath) {
|
|
||||||
try {
|
|
||||||
// First check if the path is a directory
|
|
||||||
const stats = await fs.stat(filePath);
|
|
||||||
if (stats.isDirectory()) {
|
|
||||||
throw new Error(`EISDIR: illegal operation on a directory`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check by file extension
|
|
||||||
const binaryExtensions = [
|
|
||||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.ico', '.svg',
|
|
||||||
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
|
|
||||||
'.zip', '.tar', '.gz', '.rar', '.7z',
|
|
||||||
'.exe', '.dll', '.so', '.dylib',
|
|
||||||
'.mp3', '.mp4', '.avi', '.mov', '.wav',
|
|
||||||
'.ttf', '.otf', '.woff', '.woff2',
|
|
||||||
'.bin', '.dat', '.db', '.sqlite'
|
|
||||||
];
|
|
||||||
|
|
||||||
const ext = path.extname(filePath).toLowerCase();
|
|
||||||
if (binaryExtensions.includes(ext)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// For files without clear extensions, try to read a small sample
|
|
||||||
if (stats.size === 0) {
|
|
||||||
return false; // Empty files are considered text
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read first 1024 bytes to check for null bytes
|
|
||||||
const sampleSize = Math.min(1024, stats.size);
|
|
||||||
const buffer = await fs.readFile(filePath, { encoding: null, flag: 'r' });
|
|
||||||
const sample = buffer.slice(0, sampleSize);
|
|
||||||
// If we find null bytes, it's likely binary
|
|
||||||
return sample.includes(0);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`Warning: Could not determine if file is binary: ${filePath} - ${error.message}`);
|
|
||||||
return false; // Default to text if we can't determine
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read and aggregate content from text files
|
* Read and aggregate content from text files
|
||||||
@@ -261,68 +41,6 @@ async function isBinaryFile(filePath) {
|
|||||||
* @param {Object} spinner - Optional spinner instance for progress display
|
* @param {Object} spinner - Optional spinner instance for progress display
|
||||||
* @returns {Promise<Object>} Object containing file contents and metadata
|
* @returns {Promise<Object>} Object containing file contents and metadata
|
||||||
*/
|
*/
|
||||||
async function aggregateFileContents(files, rootDir, spinner = null) {
|
|
||||||
const results = {
|
|
||||||
textFiles: [],
|
|
||||||
binaryFiles: [],
|
|
||||||
errors: [],
|
|
||||||
totalFiles: files.length,
|
|
||||||
processedFiles: 0
|
|
||||||
};
|
|
||||||
|
|
||||||
for (const filePath of files) {
|
|
||||||
try {
|
|
||||||
const relativePath = path.relative(rootDir, filePath);
|
|
||||||
|
|
||||||
// Update progress indicator
|
|
||||||
if (spinner) {
|
|
||||||
spinner.text = `Processing file ${results.processedFiles + 1}/${results.totalFiles}: ${relativePath}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const isBinary = await isBinaryFile(filePath);
|
|
||||||
|
|
||||||
if (isBinary) {
|
|
||||||
results.binaryFiles.push({
|
|
||||||
path: relativePath,
|
|
||||||
absolutePath: filePath,
|
|
||||||
size: (await fs.stat(filePath)).size
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
// Read text file content
|
|
||||||
const content = await fs.readFile(filePath, 'utf8');
|
|
||||||
results.textFiles.push({
|
|
||||||
path: relativePath,
|
|
||||||
absolutePath: filePath,
|
|
||||||
content: content,
|
|
||||||
size: content.length,
|
|
||||||
lines: content.split('\n').length
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
results.processedFiles++;
|
|
||||||
} catch (error) {
|
|
||||||
const relativePath = path.relative(rootDir, filePath);
|
|
||||||
const errorInfo = {
|
|
||||||
path: relativePath,
|
|
||||||
absolutePath: filePath,
|
|
||||||
error: error.message
|
|
||||||
};
|
|
||||||
|
|
||||||
results.errors.push(errorInfo);
|
|
||||||
|
|
||||||
// Log warning without interfering with spinner
|
|
||||||
if (spinner) {
|
|
||||||
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
|
||||||
} else {
|
|
||||||
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
results.processedFiles++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate XML output with aggregated file contents using streaming
|
* Generate XML output with aggregated file contents using streaming
|
||||||
@@ -330,111 +48,6 @@ async function aggregateFileContents(files, rootDir, spinner = null) {
|
|||||||
* @param {string} outputPath - The output file path
|
* @param {string} outputPath - The output file path
|
||||||
* @returns {Promise<void>} Promise that resolves when writing is complete
|
* @returns {Promise<void>} Promise that resolves when writing is complete
|
||||||
*/
|
*/
|
||||||
async function generateXMLOutput(aggregatedContent, outputPath) {
|
|
||||||
const { textFiles } = aggregatedContent;
|
|
||||||
|
|
||||||
// Create write stream for efficient memory usage
|
|
||||||
const writeStream = fs.createWriteStream(outputPath, { encoding: 'utf8' });
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
writeStream.on('error', reject);
|
|
||||||
writeStream.on('finish', resolve);
|
|
||||||
|
|
||||||
// Write XML header
|
|
||||||
writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
||||||
writeStream.write('<files>\n');
|
|
||||||
|
|
||||||
// Process files one by one to minimize memory usage
|
|
||||||
let fileIndex = 0;
|
|
||||||
|
|
||||||
const writeNextFile = () => {
|
|
||||||
if (fileIndex >= textFiles.length) {
|
|
||||||
// All files processed, close XML and stream
|
|
||||||
writeStream.write('</files>\n');
|
|
||||||
writeStream.end();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const file = textFiles[fileIndex];
|
|
||||||
fileIndex++;
|
|
||||||
|
|
||||||
// Write file opening tag
|
|
||||||
writeStream.write(` <file path="${escapeXml(file.path)}">`);
|
|
||||||
|
|
||||||
// Use CDATA for code content, handling CDATA end sequences properly
|
|
||||||
if (file.content?.trim()) {
|
|
||||||
const indentedContent = indentFileContent(file.content);
|
|
||||||
if (file.content.includes(']]>')) {
|
|
||||||
// If content contains ]]>, split it and wrap each part in CDATA
|
|
||||||
writeStream.write(splitAndWrapCDATA(indentedContent));
|
|
||||||
} else {
|
|
||||||
writeStream.write(`<![CDATA[\n${indentedContent}\n ]]>`);
|
|
||||||
}
|
|
||||||
} else if (file.content) {
|
|
||||||
// Handle empty or whitespace-only content
|
|
||||||
const indentedContent = indentFileContent(file.content);
|
|
||||||
writeStream.write(`<![CDATA[\n${indentedContent}\n ]]>`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write file closing tag
|
|
||||||
writeStream.write('</file>\n');
|
|
||||||
|
|
||||||
// Continue with next file on next tick to avoid stack overflow
|
|
||||||
setImmediate(writeNextFile);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Start processing files
|
|
||||||
writeNextFile();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Escape XML special characters for attributes
|
|
||||||
* @param {string} str - String to escape
|
|
||||||
* @returns {string} Escaped string
|
|
||||||
*/
|
|
||||||
function escapeXml(str) {
|
|
||||||
if (typeof str !== 'string') {
|
|
||||||
return String(str);
|
|
||||||
}
|
|
||||||
return str
|
|
||||||
.replace(/&/g, '&')
|
|
||||||
.replace(/</g, '<')
|
|
||||||
.replace(/>/g, '>')
|
|
||||||
.replace(/"/g, '"')
|
|
||||||
.replace(/'/g, ''');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Indent file content with 4 spaces for each line
|
|
||||||
* @param {string} content - Content to indent
|
|
||||||
* @returns {string} Indented content
|
|
||||||
*/
|
|
||||||
function indentFileContent(content) {
|
|
||||||
if (typeof content !== 'string') {
|
|
||||||
return String(content);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Split content into lines and add 4 spaces of indentation to each line
|
|
||||||
return content.split('\n').map(line => ` ${line}`).join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Split content containing ]]> and wrap each part in CDATA
|
|
||||||
* @param {string} content - Content to process
|
|
||||||
* @returns {string} Content with properly wrapped CDATA sections
|
|
||||||
*/
|
|
||||||
function splitAndWrapCDATA(content) {
|
|
||||||
if (typeof content !== 'string') {
|
|
||||||
return String(content);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Replace ]]> with ]]]]><![CDATA[> to escape it within CDATA
|
|
||||||
const escapedContent = content.replace(/]]>/g, ']]]]><![CDATA[>');
|
|
||||||
return `<![CDATA[
|
|
||||||
${escapedContent}
|
|
||||||
]]>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate statistics for the processed files
|
* Calculate statistics for the processed files
|
||||||
@@ -442,38 +55,6 @@ ${escapedContent}
|
|||||||
* @param {number} xmlFileSize - The size of the generated XML file in bytes
|
* @param {number} xmlFileSize - The size of the generated XML file in bytes
|
||||||
* @returns {Object} Statistics object
|
* @returns {Object} Statistics object
|
||||||
*/
|
*/
|
||||||
function calculateStatistics(aggregatedContent, xmlFileSize) {
|
|
||||||
const { textFiles, binaryFiles, errors } = aggregatedContent;
|
|
||||||
|
|
||||||
// Calculate total file size in bytes
|
|
||||||
const totalTextSize = textFiles.reduce((sum, file) => sum + file.size, 0);
|
|
||||||
const totalBinarySize = binaryFiles.reduce((sum, file) => sum + file.size, 0);
|
|
||||||
const totalSize = totalTextSize + totalBinarySize;
|
|
||||||
|
|
||||||
// Calculate total lines of code
|
|
||||||
const totalLines = textFiles.reduce((sum, file) => sum + file.lines, 0);
|
|
||||||
|
|
||||||
// Estimate token count (rough approximation: 1 token ≈ 4 characters)
|
|
||||||
const estimatedTokens = Math.ceil(xmlFileSize / 4);
|
|
||||||
|
|
||||||
// Format file size
|
|
||||||
const formatSize = (bytes) => {
|
|
||||||
if (bytes < 1024) return `${bytes} B`;
|
|
||||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
||||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
||||||
};
|
|
||||||
|
|
||||||
return {
|
|
||||||
totalFiles: textFiles.length + binaryFiles.length,
|
|
||||||
textFiles: textFiles.length,
|
|
||||||
binaryFiles: binaryFiles.length,
|
|
||||||
errorFiles: errors.length,
|
|
||||||
totalSize: formatSize(totalSize),
|
|
||||||
xmlSize: formatSize(xmlFileSize),
|
|
||||||
totalLines,
|
|
||||||
estimatedTokens: estimatedTokens.toLocaleString()
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Filter files based on .gitignore patterns
|
* Filter files based on .gitignore patterns
|
||||||
@@ -481,111 +62,80 @@ function calculateStatistics(aggregatedContent, xmlFileSize) {
|
|||||||
* @param {string} rootDir - The root directory
|
* @param {string} rootDir - The root directory
|
||||||
* @returns {Promise<string[]>} Filtered array of file paths
|
* @returns {Promise<string[]>} Filtered array of file paths
|
||||||
*/
|
*/
|
||||||
async function filterFiles(files, rootDir) {
|
|
||||||
const gitignorePath = path.join(rootDir, '.gitignore');
|
|
||||||
const ignorePatterns = await parseGitignore(gitignorePath);
|
|
||||||
|
|
||||||
// Add explicit patterns for common directories and files to ignore
|
/**
|
||||||
const additionalPatterns = [
|
* Attempt to find the project root by walking up from startDir
|
||||||
// Virtual environments
|
* Looks for common project markers like .git, package.json, pyproject.toml, etc.
|
||||||
'**/.venv/**', '**/venv/**', '**/env/**', '**/virtualenv/**',
|
* @param {string} startDir
|
||||||
'.venv/**', 'venv/**', 'env/**', 'virtualenv/**',
|
* @returns {Promise<string|null>} project root directory or null if not found
|
||||||
'.venv', 'venv', 'env', 'virtualenv',
|
*/
|
||||||
|
|
||||||
// Node modules
|
|
||||||
'**/node_modules/**',
|
|
||||||
'node_modules/**',
|
|
||||||
'node_modules',
|
|
||||||
|
|
||||||
// Python cache
|
|
||||||
'**/__pycache__/**',
|
|
||||||
'__pycache__/**',
|
|
||||||
'__pycache__',
|
|
||||||
'**/*.pyc',
|
|
||||||
'**/*.pyo',
|
|
||||||
'**/*.pyd',
|
|
||||||
|
|
||||||
// Binary and media files
|
|
||||||
'**/*.jpg', '**/*.jpeg', '**/*.png', '**/*.gif', '**/*.bmp', '**/*.ico', '**/*.svg',
|
|
||||||
'**/*.pdf', '**/*.doc', '**/*.docx', '**/*.xls', '**/*.xlsx', '**/*.ppt', '**/*.pptx',
|
|
||||||
'**/*.zip', '**/*.tar', '**/*.gz', '**/*.rar', '**/*.7z',
|
|
||||||
'**/*.exe', '**/*.dll', '**/*.so', '**/*.dylib',
|
|
||||||
'**/*.mp3', '**/*.mp4', '**/*.avi', '**/*.mov', '**/*.wav',
|
|
||||||
'**/*.ttf', '**/*.otf', '**/*.woff', '**/*.woff2'
|
|
||||||
];
|
|
||||||
|
|
||||||
const allIgnorePatterns = [
|
|
||||||
...ignorePatterns,
|
|
||||||
...additionalPatterns
|
|
||||||
];
|
|
||||||
|
|
||||||
// Convert absolute paths to relative for pattern matching
|
|
||||||
const relativeFiles = files.map(file => path.relative(rootDir, file));
|
|
||||||
|
|
||||||
// Separate positive and negative patterns
|
|
||||||
const positivePatterns = allIgnorePatterns.filter(p => !p.startsWith('!'));
|
|
||||||
const negativePatterns = allIgnorePatterns.filter(p => p.startsWith('!')).map(p => p.slice(1));
|
|
||||||
|
|
||||||
// Filter out files that match ignore patterns
|
|
||||||
const filteredRelative = [];
|
|
||||||
|
|
||||||
for (const file of relativeFiles) {
|
|
||||||
let shouldIgnore = false;
|
|
||||||
|
|
||||||
// First, explicit check for commonly ignored directories and file types
|
|
||||||
if (
|
|
||||||
// Check for virtual environments
|
|
||||||
file.includes('/.venv/') || file.includes('/venv/') ||
|
|
||||||
file.startsWith('.venv/') || file.startsWith('venv/') ||
|
|
||||||
// Check for node_modules
|
|
||||||
file.includes('/node_modules/') || file.startsWith('node_modules/') ||
|
|
||||||
// Check for Python cache
|
|
||||||
file.includes('/__pycache__/') || file.startsWith('__pycache__/') ||
|
|
||||||
file.endsWith('.pyc') || file.endsWith('.pyo') || file.endsWith('.pyd') ||
|
|
||||||
// Check for common binary file extensions
|
|
||||||
/\.(jpg|jpeg|png|gif|bmp|ico|svg|pdf|doc|docx|xls|xlsx|ppt|pptx|zip|tar|gz|rar|7z|exe|dll|so|dylib|mp3|mp4|avi|mov|wav|ttf|otf|woff|woff2)$/i.test(file)
|
|
||||||
) {
|
|
||||||
shouldIgnore = true;
|
|
||||||
} else {
|
|
||||||
// Check against other patterns
|
|
||||||
for (const pattern of positivePatterns) {
|
|
||||||
if (minimatch(file, pattern, { dot: true })) {
|
|
||||||
shouldIgnore = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then check negative patterns (don't ignore these files even if they match positive patterns)
|
|
||||||
if (shouldIgnore) {
|
|
||||||
for (const pattern of negativePatterns) {
|
|
||||||
if (minimatch(file, pattern, { dot: true })) {
|
|
||||||
shouldIgnore = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!shouldIgnore) {
|
|
||||||
filteredRelative.push(file);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert back to absolute paths
|
|
||||||
return filteredRelative.map(file => path.resolve(rootDir, file));
|
|
||||||
}
|
|
||||||
|
|
||||||
const program = new Command();
|
const program = new Command();
|
||||||
|
|
||||||
program
|
program
|
||||||
.name('bmad-flatten')
|
.name("bmad-flatten")
|
||||||
.description('BMad-Method codebase flattener tool')
|
.description("BMad-Method codebase flattener tool")
|
||||||
.version('1.0.0')
|
.version("1.0.0")
|
||||||
.option('-i, --input <path>', 'Input directory to flatten', process.cwd())
|
.option("-i, --input <path>", "Input directory to flatten", process.cwd())
|
||||||
.option('-o, --output <path>', 'Output file path', 'flattened-codebase.xml')
|
.option("-o, --output <path>", "Output file path", "flattened-codebase.xml")
|
||||||
.action(async (options) => {
|
.action(async (options) => {
|
||||||
const inputDir = path.resolve(options.input);
|
let inputDir = path.resolve(options.input);
|
||||||
const outputPath = path.resolve(options.output);
|
let outputPath = path.resolve(options.output);
|
||||||
|
|
||||||
|
// Detect if user explicitly provided -i/--input or -o/--output
|
||||||
|
const argv = process.argv.slice(2);
|
||||||
|
const userSpecifiedInput = argv.some((a) =>
|
||||||
|
a === "-i" || a === "--input" || a.startsWith("--input=")
|
||||||
|
);
|
||||||
|
const userSpecifiedOutput = argv.some((a) =>
|
||||||
|
a === "-o" || a === "--output" || a.startsWith("--output=")
|
||||||
|
);
|
||||||
|
const noPathArgs = !userSpecifiedInput && !userSpecifiedOutput;
|
||||||
|
|
||||||
|
if (noPathArgs) {
|
||||||
|
const detectedRoot = await findProjectRoot(process.cwd());
|
||||||
|
const suggestedOutput = detectedRoot
|
||||||
|
? path.join(detectedRoot, "flattened-codebase.xml")
|
||||||
|
: path.resolve("flattened-codebase.xml");
|
||||||
|
|
||||||
|
if (detectedRoot) {
|
||||||
|
const useDefaults = await promptYesNo(
|
||||||
|
`Detected project root at "${detectedRoot}". Use it as input and write output to "${suggestedOutput}"?`,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (useDefaults) {
|
||||||
|
inputDir = detectedRoot;
|
||||||
|
outputPath = suggestedOutput;
|
||||||
|
} else {
|
||||||
|
inputDir = await promptPath(
|
||||||
|
"Enter input directory path",
|
||||||
|
process.cwd(),
|
||||||
|
);
|
||||||
|
outputPath = await promptPath(
|
||||||
|
"Enter output file path",
|
||||||
|
path.join(inputDir, "flattened-codebase.xml"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log("Could not auto-detect a project root.");
|
||||||
|
inputDir = await promptPath(
|
||||||
|
"Enter input directory path",
|
||||||
|
process.cwd(),
|
||||||
|
);
|
||||||
|
outputPath = await promptPath(
|
||||||
|
"Enter output file path",
|
||||||
|
path.join(inputDir, "flattened-codebase.xml"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.error(
|
||||||
|
"Could not auto-detect a project root and no arguments were provided. Please specify -i/--input and -o/--output.",
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure output directory exists
|
||||||
|
await fs.ensureDir(path.dirname(outputPath));
|
||||||
|
|
||||||
console.log(`Flattening codebase from: ${inputDir}`);
|
console.log(`Flattening codebase from: ${inputDir}`);
|
||||||
console.log(`Output file: ${outputPath}`);
|
console.log(`Output file: ${outputPath}`);
|
||||||
@@ -598,30 +148,27 @@ program
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Import ora dynamically
|
// Import ora dynamically
|
||||||
const { default: ora } = await import('ora');
|
const { default: ora } = await import("ora");
|
||||||
|
|
||||||
// Start file discovery with spinner
|
// Start file discovery with spinner
|
||||||
const discoverySpinner = ora('🔍 Discovering files...').start();
|
const discoverySpinner = ora("🔍 Discovering files...").start();
|
||||||
const files = await discoverFiles(inputDir);
|
const files = await discoverFiles(inputDir);
|
||||||
const filteredFiles = await filterFiles(files, inputDir);
|
const filteredFiles = await filterFiles(files, inputDir);
|
||||||
discoverySpinner.succeed(`📁 Found ${filteredFiles.length} files to include`);
|
discoverySpinner.succeed(
|
||||||
|
`📁 Found ${filteredFiles.length} files to include`,
|
||||||
// Write filteredFiles to temp.txt for debugging XML including unneeded files
|
);
|
||||||
// const tempFilePath = path.join(process.cwd(), 'temp-filtered-files.txt');
|
|
||||||
// await fs.writeFile(
|
|
||||||
// tempFilePath,
|
|
||||||
// filteredFiles.map(file => `${file}\n${path.relative(inputDir, file)}\n---\n`).join('\n')
|
|
||||||
// );
|
|
||||||
// console.log(`📄 Filtered files written to: ${tempFilePath}`);
|
|
||||||
|
|
||||||
// Process files with progress tracking
|
// Process files with progress tracking
|
||||||
console.log('Reading file contents');
|
console.log("Reading file contents");
|
||||||
const processingSpinner = ora('📄 Processing files...').start();
|
const processingSpinner = ora("📄 Processing files...").start();
|
||||||
const aggregatedContent = await aggregateFileContents(filteredFiles, inputDir, processingSpinner);
|
const aggregatedContent = await aggregateFileContents(
|
||||||
processingSpinner.succeed(`✅ Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
filteredFiles,
|
||||||
|
inputDir,
|
||||||
// Log processing results for test validation
|
processingSpinner,
|
||||||
console.log(`Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
);
|
||||||
|
processingSpinner.succeed(
|
||||||
|
`✅ Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`,
|
||||||
|
);
|
||||||
if (aggregatedContent.errors.length > 0) {
|
if (aggregatedContent.errors.length > 0) {
|
||||||
console.log(`Errors: ${aggregatedContent.errors.length}`);
|
console.log(`Errors: ${aggregatedContent.errors.length}`);
|
||||||
}
|
}
|
||||||
@@ -631,27 +178,34 @@ program
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Generate XML output using streaming
|
// Generate XML output using streaming
|
||||||
const xmlSpinner = ora('🔧 Generating XML output...').start();
|
const xmlSpinner = ora("🔧 Generating XML output...").start();
|
||||||
await generateXMLOutput(aggregatedContent, outputPath);
|
await generateXMLOutput(aggregatedContent, outputPath);
|
||||||
xmlSpinner.succeed('📝 XML generation completed');
|
xmlSpinner.succeed("📝 XML generation completed");
|
||||||
|
|
||||||
// Calculate and display statistics
|
// Calculate and display statistics
|
||||||
const outputStats = await fs.stat(outputPath);
|
const outputStats = await fs.stat(outputPath);
|
||||||
const stats = calculateStatistics(aggregatedContent, outputStats.size);
|
const stats = calculateStatistics(aggregatedContent, outputStats.size);
|
||||||
|
|
||||||
// Display completion summary
|
// Display completion summary
|
||||||
console.log('\n📊 Completion Summary:');
|
console.log("\n📊 Completion Summary:");
|
||||||
console.log(`✅ Successfully processed ${filteredFiles.length} files into ${path.basename(outputPath)}`);
|
console.log(
|
||||||
|
`✅ Successfully processed ${filteredFiles.length} files into ${
|
||||||
|
path.basename(outputPath)
|
||||||
|
}`,
|
||||||
|
);
|
||||||
console.log(`📁 Output file: ${outputPath}`);
|
console.log(`📁 Output file: ${outputPath}`);
|
||||||
console.log(`📏 Total source size: ${stats.totalSize}`);
|
console.log(`📏 Total source size: ${stats.totalSize}`);
|
||||||
console.log(`📄 Generated XML size: ${stats.xmlSize}`);
|
console.log(`📄 Generated XML size: ${stats.xmlSize}`);
|
||||||
console.log(`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`);
|
console.log(
|
||||||
|
`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`,
|
||||||
|
);
|
||||||
console.log(`🔢 Estimated tokens: ${stats.estimatedTokens}`);
|
console.log(`🔢 Estimated tokens: ${stats.estimatedTokens}`);
|
||||||
console.log(`📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`);
|
console.log(
|
||||||
|
`📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`,
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('❌ Critical error:', error.message);
|
console.error("❌ Critical error:", error.message);
|
||||||
console.error('An unexpected error occurred.');
|
console.error("An unexpected error occurred.");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
45
tools/flattener/projectRoot.js
Normal file
45
tools/flattener/projectRoot.js
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
const fs = require("fs-extra");
|
||||||
|
const path = require("node:path");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to find the project root by walking up from startDir
|
||||||
|
* Looks for common project markers like .git, package.json, pyproject.toml, etc.
|
||||||
|
* @param {string} startDir
|
||||||
|
* @returns {Promise<string|null>} project root directory or null if not found
|
||||||
|
*/
|
||||||
|
async function findProjectRoot(startDir) {
|
||||||
|
try {
|
||||||
|
let dir = path.resolve(startDir);
|
||||||
|
const root = path.parse(dir).root;
|
||||||
|
const markers = [
|
||||||
|
".git",
|
||||||
|
"package.json",
|
||||||
|
"pnpm-workspace.yaml",
|
||||||
|
"yarn.lock",
|
||||||
|
"pnpm-lock.yaml",
|
||||||
|
"pyproject.toml",
|
||||||
|
"requirements.txt",
|
||||||
|
"go.mod",
|
||||||
|
"Cargo.toml",
|
||||||
|
"composer.json",
|
||||||
|
".hg",
|
||||||
|
".svn",
|
||||||
|
];
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const exists = await Promise.all(
|
||||||
|
markers.map((m) => fs.pathExists(path.join(dir, m))),
|
||||||
|
);
|
||||||
|
if (exists.some(Boolean)) {
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
if (dir === root) break;
|
||||||
|
dir = path.dirname(dir);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { findProjectRoot };
|
||||||
44
tools/flattener/prompts.js
Normal file
44
tools/flattener/prompts.js
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
const os = require("node:os");
|
||||||
|
const path = require("node:path");
|
||||||
|
const readline = require("node:readline");
|
||||||
|
const process = require("node:process");
|
||||||
|
|
||||||
|
function expandHome(p) {
|
||||||
|
if (!p) return p;
|
||||||
|
if (p.startsWith("~")) return path.join(os.homedir(), p.slice(1));
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRl() {
|
||||||
|
return readline.createInterface({
|
||||||
|
input: process.stdin,
|
||||||
|
output: process.stdout,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function promptQuestion(question) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const rl = createRl();
|
||||||
|
rl.question(question, (answer) => {
|
||||||
|
rl.close();
|
||||||
|
resolve(answer);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function promptYesNo(question, defaultYes = true) {
|
||||||
|
const suffix = defaultYes ? " [Y/n] " : " [y/N] ";
|
||||||
|
const ans = (await promptQuestion(`${question}${suffix}`)).trim().toLowerCase();
|
||||||
|
if (!ans) return defaultYes;
|
||||||
|
if (["y", "yes"].includes(ans)) return true;
|
||||||
|
if (["n", "no"].includes(ans)) return false;
|
||||||
|
return promptYesNo(question, defaultYes);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function promptPath(question, defaultValue) {
|
||||||
|
const prompt = `${question}${defaultValue ? ` (default: ${defaultValue})` : ""}: `;
|
||||||
|
const ans = (await promptQuestion(prompt)).trim();
|
||||||
|
return expandHome(ans || defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { promptYesNo, promptPath, promptQuestion, expandHome };
|
||||||
30
tools/flattener/stats.js
Normal file
30
tools/flattener/stats.js
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
function calculateStatistics(aggregatedContent, xmlFileSize) {
|
||||||
|
const { textFiles, binaryFiles, errors } = aggregatedContent;
|
||||||
|
|
||||||
|
const totalTextSize = textFiles.reduce((sum, file) => sum + file.size, 0);
|
||||||
|
const totalBinarySize = binaryFiles.reduce((sum, file) => sum + file.size, 0);
|
||||||
|
const totalSize = totalTextSize + totalBinarySize;
|
||||||
|
|
||||||
|
const totalLines = textFiles.reduce((sum, file) => sum + file.lines, 0);
|
||||||
|
|
||||||
|
const estimatedTokens = Math.ceil(xmlFileSize / 4);
|
||||||
|
|
||||||
|
const formatSize = (bytes) => {
|
||||||
|
if (bytes < 1024) return `${bytes} B`;
|
||||||
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||||
|
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
totalFiles: textFiles.length + binaryFiles.length,
|
||||||
|
textFiles: textFiles.length,
|
||||||
|
binaryFiles: binaryFiles.length,
|
||||||
|
errorFiles: errors.length,
|
||||||
|
totalSize: formatSize(totalSize),
|
||||||
|
xmlSize: formatSize(xmlFileSize),
|
||||||
|
totalLines,
|
||||||
|
estimatedTokens: estimatedTokens.toLocaleString(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { calculateStatistics };
|
||||||
86
tools/flattener/xml.js
Normal file
86
tools/flattener/xml.js
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
const fs = require("fs-extra");
|
||||||
|
|
||||||
|
function escapeXml(str) {
|
||||||
|
if (typeof str !== "string") {
|
||||||
|
return String(str);
|
||||||
|
}
|
||||||
|
return str
|
||||||
|
.replace(/&/g, "&")
|
||||||
|
.replace(/</g, "<")
|
||||||
|
.replace(/'/g, "'");
|
||||||
|
}
|
||||||
|
|
||||||
|
function indentFileContent(content) {
|
||||||
|
if (typeof content !== "string") {
|
||||||
|
return String(content);
|
||||||
|
}
|
||||||
|
return content.split("\n").map((line) => ` ${line}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function generateXMLOutput(aggregatedContent, outputPath) {
|
||||||
|
const { textFiles } = aggregatedContent;
|
||||||
|
const writeStream = fs.createWriteStream(outputPath, { encoding: "utf8" });
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
writeStream.on("error", reject);
|
||||||
|
writeStream.on("finish", resolve);
|
||||||
|
|
||||||
|
writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
|
||||||
|
writeStream.write("<files>\n");
|
||||||
|
|
||||||
|
// Sort files by path for deterministic order
|
||||||
|
const filesSorted = [...textFiles].sort((a, b) =>
|
||||||
|
a.path.localeCompare(b.path)
|
||||||
|
);
|
||||||
|
let index = 0;
|
||||||
|
|
||||||
|
const writeNext = () => {
|
||||||
|
if (index >= filesSorted.length) {
|
||||||
|
writeStream.write("</files>\n");
|
||||||
|
writeStream.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = filesSorted[index++];
|
||||||
|
const p = escapeXml(file.path);
|
||||||
|
const content = typeof file.content === "string" ? file.content : "";
|
||||||
|
|
||||||
|
if (content.length === 0) {
|
||||||
|
writeStream.write(`\t<file path='${p}'/>\n`);
|
||||||
|
setTimeout(writeNext, 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const needsCdata = content.includes("<") || content.includes("&") ||
|
||||||
|
content.includes("]]>");
|
||||||
|
if (needsCdata) {
|
||||||
|
// Open tag and CDATA on their own line with tab indent; content lines indented with two tabs
|
||||||
|
writeStream.write(`\t<file path='${p}'><![CDATA[\n`);
|
||||||
|
// Safely split any occurrences of "]]>" inside content, trim trailing newlines, indent each line with two tabs
|
||||||
|
const safe = content.replace(/]]>/g, "]]]]><![CDATA[>");
|
||||||
|
const trimmed = safe.replace(/[\r\n]+$/, "");
|
||||||
|
const indented = trimmed.length > 0
|
||||||
|
? trimmed.split("\n").map((line) => `\t\t${line}`).join("\n")
|
||||||
|
: "";
|
||||||
|
writeStream.write(indented);
|
||||||
|
// Close CDATA and attach closing tag directly after the last content line
|
||||||
|
writeStream.write("]]></file>\n");
|
||||||
|
} else {
|
||||||
|
// Write opening tag then newline; indent content with two tabs; attach closing tag directly after last content char
|
||||||
|
writeStream.write(`\t<file path='${p}'>\n`);
|
||||||
|
const trimmed = content.replace(/[\r\n]+$/, "");
|
||||||
|
const indented = trimmed.length > 0
|
||||||
|
? trimmed.split("\n").map((line) => `\t\t${line}`).join("\n")
|
||||||
|
: "";
|
||||||
|
writeStream.write(indented);
|
||||||
|
writeStream.write(`</file>\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
setTimeout(writeNext, 0);
|
||||||
|
};
|
||||||
|
|
||||||
|
writeNext();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { generateXMLOutput };
|
||||||
105
tools/shared/bannerArt.js
Normal file
105
tools/shared/bannerArt.js
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
// ASCII banner art definitions extracted from banners.js to separate art from logic
|
||||||
|
|
||||||
|
const BMAD_TITLE = "BMAD-METHOD";
|
||||||
|
const FLATTENER_TITLE = "FLATTENER";
|
||||||
|
const INSTALLER_TITLE = "INSTALLER";
|
||||||
|
|
||||||
|
// Large ASCII blocks (block-style fonts)
|
||||||
|
const BMAD_LARGE = `
|
||||||
|
██████╗ ███╗ ███╗ █████╗ ██████╗ ███╗ ███╗███████╗████████╗██╗ ██╗ ██████╗ ██████╗
|
||||||
|
██╔══██╗████╗ ████║██╔══██╗██╔══██╗ ████╗ ████║██╔════╝╚══██╔══╝██║ ██║██╔═══██╗██╔══██╗
|
||||||
|
██████╔╝██╔████╔██║███████║██║ ██║█████╗██╔████╔██║█████╗ ██║ ███████║██║ ██║██║ ██║
|
||||||
|
██╔══██╗██║╚██╔╝██║██╔══██║██║ ██║╚════╝██║╚██╔╝██║██╔══╝ ██║ ██╔══██║██║ ██║██║ ██║
|
||||||
|
██████╔╝██║ ╚═╝ ██║██║ ██║██████╔╝ ██║ ╚═╝ ██║███████╗ ██║ ██║ ██║╚██████╔╝██████╔╝
|
||||||
|
╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝╚══════╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═════╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
const FLATTENER_LARGE = `
|
||||||
|
███████╗██╗ █████╗ ████████╗████████╗███████╗███╗ ██╗███████╗██████╗
|
||||||
|
██╔════╝██║ ██╔══██╗╚══██╔══╝╚══██╔══╝██╔════╝████╗ ██║██╔════╝██╔══██╗
|
||||||
|
█████╗ ██║ ███████║ ██║ ██║ █████╗ ██╔██╗ ██║█████╗ ██████╔╝
|
||||||
|
██╔══╝ ██║ ██╔══██║ ██║ ██║ ██╔══╝ ██║╚██╗██║██╔══╝ ██╔══██╗
|
||||||
|
██║ ███████║██║ ██║ ██║ ██║ ███████╗██║ ╚████║███████╗██║ ██║
|
||||||
|
╚═╝ ╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
const INSTALLER_LARGE = `
|
||||||
|
██╗███╗ ██╗███████╗████████╗ █████╗ ██╗ ██╗ ███████╗██████╗
|
||||||
|
██║████╗ ██║██╔════╝╚══██╔══╝██╔══██╗██║ ██║ ██╔════╝██╔══██╗
|
||||||
|
██║██╔██╗ ██║███████╗ ██║ ███████║██║ ██║ █████╗ ██████╔╝
|
||||||
|
██║██║╚██╗██║╚════██║ ██║ ██╔══██║██║ ██║ ██╔══╝ ██╔══██╗
|
||||||
|
██║██║ ╚████║███████║ ██║ ██║ ██║███████╗███████╗███████╗██║ ██║
|
||||||
|
╚═╝╚═╝ ╚═══╝╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Curated medium/small/tiny variants (fixed art, no runtime scaling)
|
||||||
|
// Medium: bold framed title with heavy fill (high contrast, compact)
|
||||||
|
const BMAD_MEDIUM = `
|
||||||
|
███╗ █╗ █╗ ██╗ ███╗ █╗ █╗███╗█████╗█╗ █╗ ██╗ ███╗
|
||||||
|
█╔═█╗██╗ ██║█╔═█╗█╔═█╗ ██╗ ██║█╔═╝╚═█╔═╝█║ █║█╔═█╗█╔═█╗
|
||||||
|
███╔╝█╔███╔█║████║█║ █║██╗█╔███╔█║██╗ █║ ████║█║ █║█║ █║
|
||||||
|
█╔═█╗█║ █╔╝█║█╔═█║█║ █║╚═╝█║ █╔╝█║█╔╝ █║ █╔═█║█║ █║█║ █║
|
||||||
|
███╔╝█║ ╚╝ █║█║ █║███╔╝ █║ ╚╝ █║███╗ █║ █║ █║╚██╔╝███╔╝
|
||||||
|
╚══╝ ╚╝ ╚╝╚╝ ╚╝╚══╝ ╚╝ ╚╝╚══╝ ╚╝ ╚╝ ╚╝ ╚═╝ ╚══╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
const FLATTENER_MEDIUM = `
|
||||||
|
███╗█╗ ██╗ █████╗█████╗███╗█╗ █╗███╗███╗
|
||||||
|
█╔═╝█║ █╔═█╗╚═█╔═╝╚═█╔═╝█╔═╝██╗ █║█╔═╝█╔═█╗
|
||||||
|
██╗ █║ ████║ █║ █║ ██╗ █╔█╗█║██╗ ███╔╝
|
||||||
|
█╔╝ █║ █╔═█║ █║ █║ █╔╝ █║ ██║█╔╝ █╔═█╗
|
||||||
|
█║ ███║█║ █║ █║ █║ ███╗█║ █║███╗█║ █║
|
||||||
|
╚╝ ╚══╝╚╝ ╚╝ ╚╝ ╚╝ ╚══╝╚╝ ╚╝╚══╝╚╝ ╚╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
const INSTALLER_MEDIUM = `
|
||||||
|
█╗█╗ █╗████╗█████╗ ██╗ █╗ █╗ ███╗███╗
|
||||||
|
█║██╗ █║█╔══╝╚═█╔═╝█╔═█╗█║ █║ █╔═╝█╔═█╗
|
||||||
|
█║█╔█╗█║████╗ █║ ████║█║ █║ ██╗ ███╔╝
|
||||||
|
█║█║ ██║╚══█║ █║ █╔═█║█║ █║ █╔╝ █╔═█╗
|
||||||
|
█║█║ █║████║ █║ █║ █║███╗███╗███╗█║ █║
|
||||||
|
╚╝╚╝ ╚╝╚═══╝ ╚╝ ╚╝ ╚╝╚══╝╚══╝╚══╝╚╝ ╚╝
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Small: rounded box with bold rule
|
||||||
|
// Width: 30 columns total (28 inner)
|
||||||
|
const BMAD_SMALL = `
|
||||||
|
╭──────────────────────────╮
|
||||||
|
│ BMAD-METHOD │
|
||||||
|
╰──────────────────────────╯
|
||||||
|
`;
|
||||||
|
|
||||||
|
const FLATTENER_SMALL = `
|
||||||
|
╭──────────────────────────╮
|
||||||
|
│ FLATTENER │
|
||||||
|
╰──────────────────────────╯
|
||||||
|
`;
|
||||||
|
|
||||||
|
const INSTALLER_SMALL = `
|
||||||
|
╭──────────────────────────╮
|
||||||
|
│ INSTALLER │
|
||||||
|
╰──────────────────────────╯
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Tiny (compact brackets)
|
||||||
|
const BMAD_TINY = `[ BMAD-METHOD ]`;
|
||||||
|
const FLATTENER_TINY = `[ FLATTENER ]`;
|
||||||
|
const INSTALLER_TINY = `[ INSTALLER ]`;
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
BMAD_TITLE,
|
||||||
|
FLATTENER_TITLE,
|
||||||
|
INSTALLER_TITLE,
|
||||||
|
BMAD_LARGE,
|
||||||
|
FLATTENER_LARGE,
|
||||||
|
INSTALLER_LARGE,
|
||||||
|
BMAD_MEDIUM,
|
||||||
|
FLATTENER_MEDIUM,
|
||||||
|
INSTALLER_MEDIUM,
|
||||||
|
BMAD_SMALL,
|
||||||
|
FLATTENER_SMALL,
|
||||||
|
INSTALLER_SMALL,
|
||||||
|
BMAD_TINY,
|
||||||
|
FLATTENER_TINY,
|
||||||
|
INSTALLER_TINY,
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user