feat: modularize flattener tool into separate components with improved project root detection (#417)
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -3,6 +3,8 @@ node_modules/
|
||||
pnpm-lock.yaml
|
||||
bun.lock
|
||||
deno.lock
|
||||
pnpm-workspace.yaml
|
||||
package-lock.json
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
@@ -41,3 +43,5 @@ CLAUDE.md
|
||||
.bmad-creator-tools
|
||||
test-project-install/*
|
||||
sample-project/*
|
||||
flattened-codebase.xml
|
||||
|
||||
|
||||
37
README.md
37
README.md
@@ -144,7 +144,7 @@ npx bmad-method flatten --input /path/to/source --output /path/to/output/codebas
|
||||
|
||||
The tool will display progress and provide a comprehensive summary:
|
||||
|
||||
```
|
||||
```text
|
||||
📊 Completion Summary:
|
||||
✅ Successfully processed 156 files into flattened-codebase.xml
|
||||
📁 Output file: /path/to/your/project/flattened-codebase.xml
|
||||
@@ -155,7 +155,40 @@ The tool will display progress and provide a comprehensive summary:
|
||||
📊 File breakdown: 142 text, 14 binary, 0 errors
|
||||
```
|
||||
|
||||
The generated XML file contains all your project's source code in a structured format that AI models can easily parse and understand, making it perfect for code reviews, architecture discussions, or getting AI assistance with your BMad-Method projects.
|
||||
The generated XML file contains your project's text-based source files in a structured format that AI models can easily parse and understand, making it perfect for code reviews, architecture discussions, or getting AI assistance with your BMad-Method projects.
|
||||
|
||||
#### Advanced Usage & Options
|
||||
|
||||
- CLI options
|
||||
- `-i, --input <path>`: Directory to flatten. Default: current working directory or auto-detected project root when run interactively.
|
||||
- `-o, --output <path>`: Output file path. Default: `flattened-codebase.xml` in the chosen directory.
|
||||
- Interactive mode
|
||||
- If you do not pass `--input` and `--output` and the terminal is interactive (TTY), the tool will attempt to detect your project root (by looking for markers like `.git`, `package.json`, etc.) and prompt you to confirm or override the paths.
|
||||
- In non-interactive contexts (e.g., CI), it will prefer the detected root silently; otherwise it falls back to the current directory and default filename.
|
||||
- File discovery and ignoring
|
||||
- Uses `git ls-files` when inside a git repository for speed and correctness; otherwise falls back to a glob-based scan.
|
||||
- Applies your `.gitignore` plus a curated set of default ignore patterns (e.g., `node_modules`, build outputs, caches, logs, IDE folders, lockfiles, large media/binaries, `.env*`, and previously generated XML outputs).
|
||||
- Binary handling
|
||||
- Binary files are detected and excluded from the XML content. They are counted in the final summary but not embedded in the output.
|
||||
- XML format and safety
|
||||
- UTF-8 encoded file with root element `<files>`.
|
||||
- Each text file is emitted as a `<file path="relative/path">` element whose content is wrapped in `<![CDATA[ ... ]]>`.
|
||||
- The tool safely handles occurrences of `]]>` inside content by splitting the CDATA to preserve correctness.
|
||||
- File contents are preserved as-is and indented for readability inside the XML.
|
||||
- Performance
|
||||
- Concurrency is selected automatically based on your CPU and workload size. No configuration required.
|
||||
- Running inside a git repo improves discovery performance.
|
||||
|
||||
#### Minimal XML example
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<files>
|
||||
<file path="src/index.js"><![CDATA[
|
||||
// your source content
|
||||
]]></file>
|
||||
</files>
|
||||
```
|
||||
|
||||
## Documentation & Resources
|
||||
|
||||
|
||||
25202
package-lock.json
generated
25202
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -40,9 +40,9 @@
|
||||
"commander": "^14.0.0",
|
||||
"fs-extra": "^11.3.0",
|
||||
"glob": "^11.0.3",
|
||||
"ignore": "^7.0.5",
|
||||
"inquirer": "^8.2.6",
|
||||
"js-yaml": "^4.1.0",
|
||||
"minimatch": "^10.0.3",
|
||||
"ora": "^5.4.1"
|
||||
},
|
||||
"keywords": [
|
||||
|
||||
76
tools/flattener/aggregate.js
Normal file
76
tools/flattener/aggregate.js
Normal file
@@ -0,0 +1,76 @@
|
||||
const fs = require("fs-extra");
|
||||
const path = require("node:path");
|
||||
const os = require("node:os");
|
||||
const { isBinaryFile } = require("./binary.js");
|
||||
|
||||
/**
|
||||
* Aggregate file contents with bounded concurrency.
|
||||
* Returns text files, binary files (with size), and errors.
|
||||
* @param {string[]} files absolute file paths
|
||||
* @param {string} rootDir
|
||||
* @param {{ text?: string, warn?: (msg: string) => void } | null} spinner
|
||||
*/
|
||||
async function aggregateFileContents(files, rootDir, spinner = null) {
|
||||
const results = {
|
||||
textFiles: [],
|
||||
binaryFiles: [],
|
||||
errors: [],
|
||||
totalFiles: files.length,
|
||||
processedFiles: 0,
|
||||
};
|
||||
|
||||
// Automatic concurrency selection based on CPU count and workload size.
|
||||
// - Base on 2x logical CPUs, clamped to [2, 64]
|
||||
// - For very small workloads, avoid excessive parallelism
|
||||
const cpuCount = (os.cpus && Array.isArray(os.cpus()) ? os.cpus().length : (os.cpus?.length || 4));
|
||||
let concurrency = Math.min(64, Math.max(2, (Number(cpuCount) || 4) * 2));
|
||||
if (files.length > 0 && files.length < concurrency) {
|
||||
concurrency = Math.max(1, Math.min(concurrency, Math.ceil(files.length / 2)));
|
||||
}
|
||||
|
||||
async function processOne(filePath) {
|
||||
try {
|
||||
const relativePath = path.relative(rootDir, filePath);
|
||||
if (spinner) {
|
||||
spinner.text = `Processing: ${relativePath} (${results.processedFiles + 1}/${results.totalFiles})`;
|
||||
}
|
||||
|
||||
const binary = await isBinaryFile(filePath);
|
||||
if (binary) {
|
||||
const size = (await fs.stat(filePath)).size;
|
||||
results.binaryFiles.push({ path: relativePath, absolutePath: filePath, size });
|
||||
} else {
|
||||
const content = await fs.readFile(filePath, "utf8");
|
||||
results.textFiles.push({
|
||||
path: relativePath,
|
||||
absolutePath: filePath,
|
||||
content,
|
||||
size: content.length,
|
||||
lines: content.split("\n").length,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const relativePath = path.relative(rootDir, filePath);
|
||||
const errorInfo = { path: relativePath, absolutePath: filePath, error: error.message };
|
||||
results.errors.push(errorInfo);
|
||||
if (spinner) {
|
||||
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
} else {
|
||||
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
}
|
||||
} finally {
|
||||
results.processedFiles++;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < files.length; i += concurrency) {
|
||||
const slice = files.slice(i, i + concurrency);
|
||||
await Promise.all(slice.map(processOne));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
aggregateFileContents,
|
||||
};
|
||||
53
tools/flattener/binary.js
Normal file
53
tools/flattener/binary.js
Normal file
@@ -0,0 +1,53 @@
|
||||
const fsp = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const { Buffer } = require("node:buffer");
|
||||
|
||||
/**
|
||||
* Efficiently determine if a file is binary without reading the whole file.
|
||||
* - Fast path by extension for common binaries
|
||||
* - Otherwise read a small prefix and check for NUL bytes
|
||||
* @param {string} filePath
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function isBinaryFile(filePath) {
|
||||
try {
|
||||
const stats = await fsp.stat(filePath);
|
||||
if (stats.isDirectory()) {
|
||||
throw new Error("EISDIR: illegal operation on a directory");
|
||||
}
|
||||
|
||||
const binaryExtensions = new Set([
|
||||
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".ico", ".svg",
|
||||
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||
".zip", ".tar", ".gz", ".rar", ".7z",
|
||||
".exe", ".dll", ".so", ".dylib",
|
||||
".mp3", ".mp4", ".avi", ".mov", ".wav",
|
||||
".ttf", ".otf", ".woff", ".woff2",
|
||||
".bin", ".dat", ".db", ".sqlite",
|
||||
]);
|
||||
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
if (binaryExtensions.has(ext)) return true;
|
||||
if (stats.size === 0) return false;
|
||||
|
||||
const sampleSize = Math.min(4096, stats.size);
|
||||
const fd = await fsp.open(filePath, "r");
|
||||
try {
|
||||
const buffer = Buffer.allocUnsafe(sampleSize);
|
||||
const { bytesRead } = await fd.read(buffer, 0, sampleSize, 0);
|
||||
const slice = bytesRead === sampleSize ? buffer : buffer.subarray(0, bytesRead);
|
||||
return slice.includes(0);
|
||||
} finally {
|
||||
await fd.close();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Warning: Could not determine if file is binary: ${filePath} - ${error.message}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isBinaryFile,
|
||||
};
|
||||
70
tools/flattener/discovery.js
Normal file
70
tools/flattener/discovery.js
Normal file
@@ -0,0 +1,70 @@
|
||||
const path = require("node:path");
|
||||
const { execFile } = require("node:child_process");
|
||||
const { promisify } = require("node:util");
|
||||
const { glob } = require("glob");
|
||||
const { loadIgnore } = require("./ignoreRules.js");
|
||||
|
||||
const pExecFile = promisify(execFile);
|
||||
|
||||
async function isGitRepo(rootDir) {
|
||||
try {
|
||||
const { stdout } = await pExecFile("git", [
|
||||
"rev-parse",
|
||||
"--is-inside-work-tree",
|
||||
], { cwd: rootDir });
|
||||
return String(stdout || "").toString().trim() === "true";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function gitListFiles(rootDir) {
|
||||
try {
|
||||
const { stdout } = await pExecFile("git", [
|
||||
"ls-files",
|
||||
"-co",
|
||||
"--exclude-standard",
|
||||
], { cwd: rootDir });
|
||||
return String(stdout || "")
|
||||
.split(/\r?\n/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover files under rootDir.
|
||||
* - Prefer git ls-files when available for speed/correctness
|
||||
* - Fallback to glob and apply unified ignore rules
|
||||
* @param {string} rootDir
|
||||
* @param {object} [options]
|
||||
* @param {boolean} [options.preferGit=true]
|
||||
* @returns {Promise<string[]>} absolute file paths
|
||||
*/
|
||||
async function discoverFiles(rootDir, options = {}) {
|
||||
const { preferGit = true } = options;
|
||||
const { filter } = await loadIgnore(rootDir);
|
||||
|
||||
// Try git first
|
||||
if (preferGit && await isGitRepo(rootDir)) {
|
||||
const relFiles = await gitListFiles(rootDir);
|
||||
const filteredRel = relFiles.filter((p) => filter(p));
|
||||
return filteredRel.map((p) => path.resolve(rootDir, p));
|
||||
}
|
||||
|
||||
// Glob fallback
|
||||
const globbed = await glob("**/*", {
|
||||
cwd: rootDir,
|
||||
nodir: true,
|
||||
dot: true,
|
||||
follow: false,
|
||||
});
|
||||
const filteredRel = globbed.filter((p) => filter(p));
|
||||
return filteredRel.map((p) => path.resolve(rootDir, p));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
discoverFiles,
|
||||
};
|
||||
35
tools/flattener/files.js
Normal file
35
tools/flattener/files.js
Normal file
@@ -0,0 +1,35 @@
|
||||
const path = require("node:path");
|
||||
const discovery = require("./discovery.js");
|
||||
const ignoreRules = require("./ignoreRules.js");
|
||||
const { isBinaryFile } = require("./binary.js");
|
||||
const { aggregateFileContents } = require("./aggregate.js");
|
||||
|
||||
// Backward-compatible signature; delegate to central loader
|
||||
async function parseGitignore(gitignorePath) {
|
||||
return await ignoreRules.parseGitignore(gitignorePath);
|
||||
}
|
||||
|
||||
async function discoverFiles(rootDir) {
|
||||
try {
|
||||
// Delegate to discovery module which respects .gitignore and defaults
|
||||
return await discovery.discoverFiles(rootDir, { preferGit: true });
|
||||
} catch (error) {
|
||||
console.error("Error discovering files:", error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function filterFiles(files, rootDir) {
|
||||
const { filter } = await ignoreRules.loadIgnore(rootDir);
|
||||
const relativeFiles = files.map((f) => path.relative(rootDir, f));
|
||||
const filteredRelative = relativeFiles.filter((p) => filter(p));
|
||||
return filteredRelative.map((p) => path.resolve(rootDir, p));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
parseGitignore,
|
||||
discoverFiles,
|
||||
isBinaryFile,
|
||||
aggregateFileContents,
|
||||
filterFiles,
|
||||
};
|
||||
176
tools/flattener/ignoreRules.js
Normal file
176
tools/flattener/ignoreRules.js
Normal file
@@ -0,0 +1,176 @@
|
||||
const fs = require("fs-extra");
|
||||
const path = require("node:path");
|
||||
const ignore = require("ignore");
|
||||
|
||||
// Central default ignore patterns for discovery and filtering.
|
||||
// These complement .gitignore and are applied regardless of VCS presence.
|
||||
const DEFAULT_PATTERNS = [
|
||||
// Project/VCS
|
||||
"**/.bmad-core/**",
|
||||
"**/.git/**",
|
||||
"**/.svn/**",
|
||||
"**/.hg/**",
|
||||
"**/.bzr/**",
|
||||
// Package/build outputs
|
||||
"**/node_modules/**",
|
||||
"**/bower_components/**",
|
||||
"**/vendor/**",
|
||||
"**/packages/**",
|
||||
"**/build/**",
|
||||
"**/dist/**",
|
||||
"**/out/**",
|
||||
"**/target/**",
|
||||
"**/bin/**",
|
||||
"**/obj/**",
|
||||
"**/release/**",
|
||||
"**/debug/**",
|
||||
// Environments
|
||||
"**/.venv/**",
|
||||
"**/venv/**",
|
||||
"**/.virtualenv/**",
|
||||
"**/virtualenv/**",
|
||||
"**/env/**",
|
||||
// Logs & coverage
|
||||
"**/*.log",
|
||||
"**/npm-debug.log*",
|
||||
"**/yarn-debug.log*",
|
||||
"**/yarn-error.log*",
|
||||
"**/lerna-debug.log*",
|
||||
"**/coverage/**",
|
||||
"**/.nyc_output/**",
|
||||
"**/.coverage/**",
|
||||
"**/test-results/**",
|
||||
// Caches & temp
|
||||
"**/.cache/**",
|
||||
"**/.tmp/**",
|
||||
"**/.temp/**",
|
||||
"**/tmp/**",
|
||||
"**/temp/**",
|
||||
"**/.sass-cache/**",
|
||||
// IDE/editor
|
||||
"**/.vscode/**",
|
||||
"**/.idea/**",
|
||||
"**/*.swp",
|
||||
"**/*.swo",
|
||||
"**/*~",
|
||||
"**/.project",
|
||||
"**/.classpath",
|
||||
"**/.settings/**",
|
||||
"**/*.sublime-project",
|
||||
"**/*.sublime-workspace",
|
||||
// Lockfiles
|
||||
"**/package-lock.json",
|
||||
"**/yarn.lock",
|
||||
"**/pnpm-lock.yaml",
|
||||
"**/composer.lock",
|
||||
"**/Pipfile.lock",
|
||||
// Python/Java/compiled artifacts
|
||||
"**/*.pyc",
|
||||
"**/*.pyo",
|
||||
"**/*.pyd",
|
||||
"**/__pycache__/**",
|
||||
"**/*.class",
|
||||
"**/*.jar",
|
||||
"**/*.war",
|
||||
"**/*.ear",
|
||||
"**/*.o",
|
||||
"**/*.so",
|
||||
"**/*.dll",
|
||||
"**/*.exe",
|
||||
// System junk
|
||||
"**/lib64/**",
|
||||
"**/.venv/lib64/**",
|
||||
"**/venv/lib64/**",
|
||||
"**/_site/**",
|
||||
"**/.jekyll-cache/**",
|
||||
"**/.jekyll-metadata",
|
||||
"**/.DS_Store",
|
||||
"**/.DS_Store?",
|
||||
"**/._*",
|
||||
"**/.Spotlight-V100/**",
|
||||
"**/.Trashes/**",
|
||||
"**/ehthumbs.db",
|
||||
"**/Thumbs.db",
|
||||
"**/desktop.ini",
|
||||
// XML outputs
|
||||
"**/flattened-codebase.xml",
|
||||
"**/repomix-output.xml",
|
||||
// Images, media, fonts, archives, docs, dylibs
|
||||
"**/*.jpg",
|
||||
"**/*.jpeg",
|
||||
"**/*.png",
|
||||
"**/*.gif",
|
||||
"**/*.bmp",
|
||||
"**/*.ico",
|
||||
"**/*.svg",
|
||||
"**/*.pdf",
|
||||
"**/*.doc",
|
||||
"**/*.docx",
|
||||
"**/*.xls",
|
||||
"**/*.xlsx",
|
||||
"**/*.ppt",
|
||||
"**/*.pptx",
|
||||
"**/*.zip",
|
||||
"**/*.tar",
|
||||
"**/*.gz",
|
||||
"**/*.rar",
|
||||
"**/*.7z",
|
||||
"**/*.dylib",
|
||||
"**/*.mp3",
|
||||
"**/*.mp4",
|
||||
"**/*.avi",
|
||||
"**/*.mov",
|
||||
"**/*.wav",
|
||||
"**/*.ttf",
|
||||
"**/*.otf",
|
||||
"**/*.woff",
|
||||
"**/*.woff2",
|
||||
// Env files
|
||||
"**/.env",
|
||||
"**/.env.*",
|
||||
"**/*.env",
|
||||
// Misc
|
||||
"**/junit.xml",
|
||||
];
|
||||
|
||||
async function readIgnoreFile(filePath) {
|
||||
try {
|
||||
if (!await fs.pathExists(filePath)) return [];
|
||||
const content = await fs.readFile(filePath, "utf8");
|
||||
return content
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l && !l.startsWith("#"));
|
||||
} catch (err) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Backward compatible export matching previous signature
|
||||
async function parseGitignore(gitignorePath) {
|
||||
return readIgnoreFile(gitignorePath);
|
||||
}
|
||||
|
||||
async function loadIgnore(rootDir, extraPatterns = []) {
|
||||
const ig = ignore();
|
||||
const gitignorePath = path.join(rootDir, ".gitignore");
|
||||
const patterns = [
|
||||
...await readIgnoreFile(gitignorePath),
|
||||
...DEFAULT_PATTERNS,
|
||||
...extraPatterns,
|
||||
];
|
||||
// De-duplicate
|
||||
const unique = Array.from(new Set(patterns.map((p) => String(p))));
|
||||
ig.add(unique);
|
||||
|
||||
// Include-only filter: return true if path should be included
|
||||
const filter = (relativePath) => !ig.ignores(relativePath.replace(/\\/g, "/"));
|
||||
|
||||
return { ig, filter, patterns: unique };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_PATTERNS,
|
||||
parseGitignore,
|
||||
loadIgnore,
|
||||
};
|
||||
@@ -1,258 +1,38 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { Command } = require('commander');
|
||||
const fs = require('fs-extra');
|
||||
const path = require('node:path');
|
||||
const { glob } = require('glob');
|
||||
const { minimatch } = require('minimatch');
|
||||
const { Command } = require("commander");
|
||||
const fs = require("fs-extra");
|
||||
const path = require("node:path");
|
||||
const process = require("node:process");
|
||||
|
||||
// Modularized components
|
||||
const { findProjectRoot } = require("./projectRoot.js");
|
||||
const { promptYesNo, promptPath } = require("./prompts.js");
|
||||
const {
|
||||
discoverFiles,
|
||||
filterFiles,
|
||||
aggregateFileContents,
|
||||
} = require("./files.js");
|
||||
const { generateXMLOutput } = require("./xml.js");
|
||||
const { calculateStatistics } = require("./stats.js");
|
||||
|
||||
/**
|
||||
* Recursively discover all files in a directory
|
||||
* @param {string} rootDir - The root directory to scan
|
||||
* @returns {Promise<string[]>} Array of file paths
|
||||
*/
|
||||
async function discoverFiles(rootDir) {
|
||||
try {
|
||||
const gitignorePath = path.join(rootDir, '.gitignore');
|
||||
const gitignorePatterns = await parseGitignore(gitignorePath);
|
||||
|
||||
// Common gitignore patterns that should always be ignored
|
||||
const commonIgnorePatterns = [
|
||||
// Version control
|
||||
'.git/**',
|
||||
'.svn/**',
|
||||
'.hg/**',
|
||||
'.bzr/**',
|
||||
|
||||
// Dependencies
|
||||
'node_modules/**',
|
||||
'bower_components/**',
|
||||
'vendor/**',
|
||||
'packages/**',
|
||||
|
||||
// Build outputs
|
||||
'build/**',
|
||||
'dist/**',
|
||||
'out/**',
|
||||
'target/**',
|
||||
'bin/**',
|
||||
'obj/**',
|
||||
'release/**',
|
||||
'debug/**',
|
||||
|
||||
// Environment and config
|
||||
'.env',
|
||||
'.env.*',
|
||||
'*.env',
|
||||
'.config',
|
||||
'.venv/**',
|
||||
'*/.venv/**',
|
||||
'**/.venv/**',
|
||||
'.venv',
|
||||
'venv/**',
|
||||
'*/venv/**',
|
||||
'**/venv/**',
|
||||
'venv',
|
||||
'env/**',
|
||||
'*/env/**',
|
||||
'**/env/**',
|
||||
'virtualenv/**',
|
||||
'*/virtualenv/**',
|
||||
'**/virtualenv/**',
|
||||
|
||||
// Logs
|
||||
'logs/**',
|
||||
'*.log',
|
||||
'npm-debug.log*',
|
||||
'yarn-debug.log*',
|
||||
'yarn-error.log*',
|
||||
'lerna-debug.log*',
|
||||
|
||||
// Coverage and testing
|
||||
'coverage/**',
|
||||
'.nyc_output/**',
|
||||
'.coverage/**',
|
||||
'test-results/**',
|
||||
'junit.xml',
|
||||
|
||||
// Cache directories
|
||||
'.cache/**',
|
||||
'.tmp/**',
|
||||
'.temp/**',
|
||||
'tmp/**',
|
||||
'temp/**',
|
||||
'.sass-cache/**',
|
||||
'.eslintcache',
|
||||
'.stylelintcache',
|
||||
|
||||
// OS generated files
|
||||
'.DS_Store',
|
||||
'.DS_Store?',
|
||||
'._*',
|
||||
'.Spotlight-V100',
|
||||
'.Trashes',
|
||||
'ehthumbs.db',
|
||||
'Thumbs.db',
|
||||
'desktop.ini',
|
||||
|
||||
// IDE and editor files
|
||||
'.vscode/**',
|
||||
'.idea/**',
|
||||
'*.swp',
|
||||
'*.swo',
|
||||
'*~',
|
||||
'.project',
|
||||
'.classpath',
|
||||
'.settings/**',
|
||||
'*.sublime-project',
|
||||
'*.sublime-workspace',
|
||||
|
||||
// Package manager files
|
||||
'package-lock.json',
|
||||
'yarn.lock',
|
||||
'pnpm-lock.yaml',
|
||||
'composer.lock',
|
||||
'Pipfile.lock',
|
||||
|
||||
// Runtime and compiled files
|
||||
'*.pyc',
|
||||
'*.pyo',
|
||||
'*.pyd',
|
||||
'__pycache__/**',
|
||||
'*.class',
|
||||
'*.jar',
|
||||
'*.war',
|
||||
'*.ear',
|
||||
'*.o',
|
||||
'*.so',
|
||||
'*.dll',
|
||||
'*.exe',
|
||||
'lib64/**',
|
||||
'**/.venv/lib64/**',
|
||||
'**/venv/lib64/**',
|
||||
|
||||
// Documentation build
|
||||
'_site/**',
|
||||
'.jekyll-cache/**',
|
||||
'.jekyll-metadata',
|
||||
|
||||
// Flattener specific outputs
|
||||
'flattened-codebase.xml',
|
||||
'repomix-output.xml'
|
||||
];
|
||||
|
||||
const combinedIgnores = [
|
||||
...gitignorePatterns,
|
||||
...commonIgnorePatterns
|
||||
];
|
||||
|
||||
// Add specific patterns for commonly ignored directories and files
|
||||
const additionalGlobIgnores = [
|
||||
// Virtual environments
|
||||
'**/.venv/**', '**/venv/**', '**/.virtualenv/**', '**/virtualenv/**',
|
||||
// Node modules
|
||||
'**/node_modules/**',
|
||||
// Python cache
|
||||
'**/__pycache__/**', '**/*.pyc', '**/*.pyo', '**/*.pyd',
|
||||
// Binary and media files
|
||||
'**/*.jpg', '**/*.jpeg', '**/*.png', '**/*.gif', '**/*.bmp', '**/*.ico', '**/*.svg',
|
||||
'**/*.pdf', '**/*.doc', '**/*.docx', '**/*.xls', '**/*.xlsx', '**/*.ppt', '**/*.pptx',
|
||||
'**/*.zip', '**/*.tar', '**/*.gz', '**/*.rar', '**/*.7z',
|
||||
'**/*.exe', '**/*.dll', '**/*.so', '**/*.dylib',
|
||||
'**/*.mp3', '**/*.mp4', '**/*.avi', '**/*.mov', '**/*.wav',
|
||||
'**/*.ttf', '**/*.otf', '**/*.woff', '**/*.woff2'
|
||||
];
|
||||
|
||||
// Use glob to recursively find all files, excluding common ignore patterns
|
||||
const files = await glob('**/*', {
|
||||
cwd: rootDir,
|
||||
nodir: true, // Only files, not directories
|
||||
dot: true, // Include hidden files
|
||||
follow: false, // Don't follow symbolic links
|
||||
ignore: [...combinedIgnores, ...additionalGlobIgnores]
|
||||
});
|
||||
|
||||
return files.map(file => path.resolve(rootDir, file));
|
||||
} catch (error) {
|
||||
console.error('Error discovering files:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse .gitignore file and return ignore patterns
|
||||
* @param {string} gitignorePath - Path to .gitignore file
|
||||
* @returns {Promise<string[]>} Array of ignore patterns
|
||||
*/
|
||||
async function parseGitignore(gitignorePath) {
|
||||
try {
|
||||
if (!await fs.pathExists(gitignorePath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const content = await fs.readFile(gitignorePath, 'utf8');
|
||||
return content
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line && !line.startsWith('#')) // Remove empty lines and comments
|
||||
.map(pattern => {
|
||||
// Convert gitignore patterns to glob patterns
|
||||
if (pattern.endsWith('/')) {
|
||||
return pattern + '**';
|
||||
}
|
||||
return pattern;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error parsing .gitignore:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file is binary using file command and heuristics
|
||||
* @param {string} filePath - Path to the file
|
||||
* @returns {Promise<boolean>} True if file is binary
|
||||
*/
|
||||
async function isBinaryFile(filePath) {
|
||||
try {
|
||||
// First check if the path is a directory
|
||||
const stats = await fs.stat(filePath);
|
||||
if (stats.isDirectory()) {
|
||||
throw new Error(`EISDIR: illegal operation on a directory`);
|
||||
}
|
||||
|
||||
// Check by file extension
|
||||
const binaryExtensions = [
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.ico', '.svg',
|
||||
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
|
||||
'.zip', '.tar', '.gz', '.rar', '.7z',
|
||||
'.exe', '.dll', '.so', '.dylib',
|
||||
'.mp3', '.mp4', '.avi', '.mov', '.wav',
|
||||
'.ttf', '.otf', '.woff', '.woff2',
|
||||
'.bin', '.dat', '.db', '.sqlite'
|
||||
];
|
||||
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
if (binaryExtensions.includes(ext)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// For files without clear extensions, try to read a small sample
|
||||
if (stats.size === 0) {
|
||||
return false; // Empty files are considered text
|
||||
}
|
||||
|
||||
// Read first 1024 bytes to check for null bytes
|
||||
const sampleSize = Math.min(1024, stats.size);
|
||||
const buffer = await fs.readFile(filePath, { encoding: null, flag: 'r' });
|
||||
const sample = buffer.slice(0, sampleSize);
|
||||
// If we find null bytes, it's likely binary
|
||||
return sample.includes(0);
|
||||
} catch (error) {
|
||||
console.warn(`Warning: Could not determine if file is binary: ${filePath} - ${error.message}`);
|
||||
return false; // Default to text if we can't determine
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and aggregate content from text files
|
||||
@@ -261,68 +41,6 @@ async function isBinaryFile(filePath) {
|
||||
* @param {Object} spinner - Optional spinner instance for progress display
|
||||
* @returns {Promise<Object>} Object containing file contents and metadata
|
||||
*/
|
||||
async function aggregateFileContents(files, rootDir, spinner = null) {
|
||||
const results = {
|
||||
textFiles: [],
|
||||
binaryFiles: [],
|
||||
errors: [],
|
||||
totalFiles: files.length,
|
||||
processedFiles: 0
|
||||
};
|
||||
|
||||
for (const filePath of files) {
|
||||
try {
|
||||
const relativePath = path.relative(rootDir, filePath);
|
||||
|
||||
// Update progress indicator
|
||||
if (spinner) {
|
||||
spinner.text = `Processing file ${results.processedFiles + 1}/${results.totalFiles}: ${relativePath}`;
|
||||
}
|
||||
|
||||
const isBinary = await isBinaryFile(filePath);
|
||||
|
||||
if (isBinary) {
|
||||
results.binaryFiles.push({
|
||||
path: relativePath,
|
||||
absolutePath: filePath,
|
||||
size: (await fs.stat(filePath)).size
|
||||
});
|
||||
} else {
|
||||
// Read text file content
|
||||
const content = await fs.readFile(filePath, 'utf8');
|
||||
results.textFiles.push({
|
||||
path: relativePath,
|
||||
absolutePath: filePath,
|
||||
content: content,
|
||||
size: content.length,
|
||||
lines: content.split('\n').length
|
||||
});
|
||||
}
|
||||
|
||||
results.processedFiles++;
|
||||
} catch (error) {
|
||||
const relativePath = path.relative(rootDir, filePath);
|
||||
const errorInfo = {
|
||||
path: relativePath,
|
||||
absolutePath: filePath,
|
||||
error: error.message
|
||||
};
|
||||
|
||||
results.errors.push(errorInfo);
|
||||
|
||||
// Log warning without interfering with spinner
|
||||
if (spinner) {
|
||||
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
} else {
|
||||
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
}
|
||||
|
||||
results.processedFiles++;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate XML output with aggregated file contents using streaming
|
||||
@@ -330,111 +48,6 @@ async function aggregateFileContents(files, rootDir, spinner = null) {
|
||||
* @param {string} outputPath - The output file path
|
||||
* @returns {Promise<void>} Promise that resolves when writing is complete
|
||||
*/
|
||||
async function generateXMLOutput(aggregatedContent, outputPath) {
|
||||
const { textFiles } = aggregatedContent;
|
||||
|
||||
// Create write stream for efficient memory usage
|
||||
const writeStream = fs.createWriteStream(outputPath, { encoding: 'utf8' });
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writeStream.on('error', reject);
|
||||
writeStream.on('finish', resolve);
|
||||
|
||||
// Write XML header
|
||||
writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
|
||||
writeStream.write('<files>\n');
|
||||
|
||||
// Process files one by one to minimize memory usage
|
||||
let fileIndex = 0;
|
||||
|
||||
const writeNextFile = () => {
|
||||
if (fileIndex >= textFiles.length) {
|
||||
// All files processed, close XML and stream
|
||||
writeStream.write('</files>\n');
|
||||
writeStream.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const file = textFiles[fileIndex];
|
||||
fileIndex++;
|
||||
|
||||
// Write file opening tag
|
||||
writeStream.write(` <file path="${escapeXml(file.path)}">`);
|
||||
|
||||
// Use CDATA for code content, handling CDATA end sequences properly
|
||||
if (file.content?.trim()) {
|
||||
const indentedContent = indentFileContent(file.content);
|
||||
if (file.content.includes(']]>')) {
|
||||
// If content contains ]]>, split it and wrap each part in CDATA
|
||||
writeStream.write(splitAndWrapCDATA(indentedContent));
|
||||
} else {
|
||||
writeStream.write(`<![CDATA[\n${indentedContent}\n ]]>`);
|
||||
}
|
||||
} else if (file.content) {
|
||||
// Handle empty or whitespace-only content
|
||||
const indentedContent = indentFileContent(file.content);
|
||||
writeStream.write(`<![CDATA[\n${indentedContent}\n ]]>`);
|
||||
}
|
||||
|
||||
// Write file closing tag
|
||||
writeStream.write('</file>\n');
|
||||
|
||||
// Continue with next file on next tick to avoid stack overflow
|
||||
setImmediate(writeNextFile);
|
||||
};
|
||||
|
||||
// Start processing files
|
||||
writeNextFile();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape XML special characters for attributes
|
||||
* @param {string} str - String to escape
|
||||
* @returns {string} Escaped string
|
||||
*/
|
||||
function escapeXml(str) {
|
||||
if (typeof str !== 'string') {
|
||||
return String(str);
|
||||
}
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
/**
|
||||
* Indent file content with 4 spaces for each line
|
||||
* @param {string} content - Content to indent
|
||||
* @returns {string} Indented content
|
||||
*/
|
||||
function indentFileContent(content) {
|
||||
if (typeof content !== 'string') {
|
||||
return String(content);
|
||||
}
|
||||
|
||||
// Split content into lines and add 4 spaces of indentation to each line
|
||||
return content.split('\n').map(line => ` ${line}`).join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Split content containing ]]> and wrap each part in CDATA
|
||||
* @param {string} content - Content to process
|
||||
* @returns {string} Content with properly wrapped CDATA sections
|
||||
*/
|
||||
function splitAndWrapCDATA(content) {
|
||||
if (typeof content !== 'string') {
|
||||
return String(content);
|
||||
}
|
||||
|
||||
// Replace ]]> with ]]]]><![CDATA[> to escape it within CDATA
|
||||
const escapedContent = content.replace(/]]>/g, ']]]]><![CDATA[>');
|
||||
return `<![CDATA[
|
||||
${escapedContent}
|
||||
]]>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate statistics for the processed files
|
||||
@@ -442,38 +55,6 @@ ${escapedContent}
|
||||
* @param {number} xmlFileSize - The size of the generated XML file in bytes
|
||||
* @returns {Object} Statistics object
|
||||
*/
|
||||
function calculateStatistics(aggregatedContent, xmlFileSize) {
|
||||
const { textFiles, binaryFiles, errors } = aggregatedContent;
|
||||
|
||||
// Calculate total file size in bytes
|
||||
const totalTextSize = textFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalBinarySize = binaryFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalSize = totalTextSize + totalBinarySize;
|
||||
|
||||
// Calculate total lines of code
|
||||
const totalLines = textFiles.reduce((sum, file) => sum + file.lines, 0);
|
||||
|
||||
// Estimate token count (rough approximation: 1 token ≈ 4 characters)
|
||||
const estimatedTokens = Math.ceil(xmlFileSize / 4);
|
||||
|
||||
// Format file size
|
||||
const formatSize = (bytes) => {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
};
|
||||
|
||||
return {
|
||||
totalFiles: textFiles.length + binaryFiles.length,
|
||||
textFiles: textFiles.length,
|
||||
binaryFiles: binaryFiles.length,
|
||||
errorFiles: errors.length,
|
||||
totalSize: formatSize(totalSize),
|
||||
xmlSize: formatSize(xmlFileSize),
|
||||
totalLines,
|
||||
estimatedTokens: estimatedTokens.toLocaleString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter files based on .gitignore patterns
|
||||
@@ -481,112 +62,81 @@ function calculateStatistics(aggregatedContent, xmlFileSize) {
|
||||
* @param {string} rootDir - The root directory
|
||||
* @returns {Promise<string[]>} Filtered array of file paths
|
||||
*/
|
||||
async function filterFiles(files, rootDir) {
|
||||
const gitignorePath = path.join(rootDir, '.gitignore');
|
||||
const ignorePatterns = await parseGitignore(gitignorePath);
|
||||
|
||||
// Add explicit patterns for common directories and files to ignore
|
||||
const additionalPatterns = [
|
||||
// Virtual environments
|
||||
'**/.venv/**', '**/venv/**', '**/env/**', '**/virtualenv/**',
|
||||
'.venv/**', 'venv/**', 'env/**', 'virtualenv/**',
|
||||
'.venv', 'venv', 'env', 'virtualenv',
|
||||
|
||||
// Node modules
|
||||
'**/node_modules/**',
|
||||
'node_modules/**',
|
||||
'node_modules',
|
||||
|
||||
// Python cache
|
||||
'**/__pycache__/**',
|
||||
'__pycache__/**',
|
||||
'__pycache__',
|
||||
'**/*.pyc',
|
||||
'**/*.pyo',
|
||||
'**/*.pyd',
|
||||
|
||||
// Binary and media files
|
||||
'**/*.jpg', '**/*.jpeg', '**/*.png', '**/*.gif', '**/*.bmp', '**/*.ico', '**/*.svg',
|
||||
'**/*.pdf', '**/*.doc', '**/*.docx', '**/*.xls', '**/*.xlsx', '**/*.ppt', '**/*.pptx',
|
||||
'**/*.zip', '**/*.tar', '**/*.gz', '**/*.rar', '**/*.7z',
|
||||
'**/*.exe', '**/*.dll', '**/*.so', '**/*.dylib',
|
||||
'**/*.mp3', '**/*.mp4', '**/*.avi', '**/*.mov', '**/*.wav',
|
||||
'**/*.ttf', '**/*.otf', '**/*.woff', '**/*.woff2'
|
||||
];
|
||||
|
||||
const allIgnorePatterns = [
|
||||
...ignorePatterns,
|
||||
...additionalPatterns
|
||||
];
|
||||
|
||||
// Convert absolute paths to relative for pattern matching
|
||||
const relativeFiles = files.map(file => path.relative(rootDir, file));
|
||||
|
||||
// Separate positive and negative patterns
|
||||
const positivePatterns = allIgnorePatterns.filter(p => !p.startsWith('!'));
|
||||
const negativePatterns = allIgnorePatterns.filter(p => p.startsWith('!')).map(p => p.slice(1));
|
||||
|
||||
// Filter out files that match ignore patterns
|
||||
const filteredRelative = [];
|
||||
|
||||
for (const file of relativeFiles) {
|
||||
let shouldIgnore = false;
|
||||
|
||||
// First, explicit check for commonly ignored directories and file types
|
||||
if (
|
||||
// Check for virtual environments
|
||||
file.includes('/.venv/') || file.includes('/venv/') ||
|
||||
file.startsWith('.venv/') || file.startsWith('venv/') ||
|
||||
// Check for node_modules
|
||||
file.includes('/node_modules/') || file.startsWith('node_modules/') ||
|
||||
// Check for Python cache
|
||||
file.includes('/__pycache__/') || file.startsWith('__pycache__/') ||
|
||||
file.endsWith('.pyc') || file.endsWith('.pyo') || file.endsWith('.pyd') ||
|
||||
// Check for common binary file extensions
|
||||
/\.(jpg|jpeg|png|gif|bmp|ico|svg|pdf|doc|docx|xls|xlsx|ppt|pptx|zip|tar|gz|rar|7z|exe|dll|so|dylib|mp3|mp4|avi|mov|wav|ttf|otf|woff|woff2)$/i.test(file)
|
||||
) {
|
||||
shouldIgnore = true;
|
||||
} else {
|
||||
// Check against other patterns
|
||||
for (const pattern of positivePatterns) {
|
||||
if (minimatch(file, pattern, { dot: true })) {
|
||||
shouldIgnore = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Then check negative patterns (don't ignore these files even if they match positive patterns)
|
||||
if (shouldIgnore) {
|
||||
for (const pattern of negativePatterns) {
|
||||
if (minimatch(file, pattern, { dot: true })) {
|
||||
shouldIgnore = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldIgnore) {
|
||||
filteredRelative.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert back to absolute paths
|
||||
return filteredRelative.map(file => path.resolve(rootDir, file));
|
||||
}
|
||||
/**
|
||||
* Attempt to find the project root by walking up from startDir
|
||||
* Looks for common project markers like .git, package.json, pyproject.toml, etc.
|
||||
* @param {string} startDir
|
||||
* @returns {Promise<string|null>} project root directory or null if not found
|
||||
*/
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('bmad-flatten')
|
||||
.description('BMad-Method codebase flattener tool')
|
||||
.version('1.0.0')
|
||||
.option('-i, --input <path>', 'Input directory to flatten', process.cwd())
|
||||
.option('-o, --output <path>', 'Output file path', 'flattened-codebase.xml')
|
||||
.name("bmad-flatten")
|
||||
.description("BMad-Method codebase flattener tool")
|
||||
.version("1.0.0")
|
||||
.option("-i, --input <path>", "Input directory to flatten", process.cwd())
|
||||
.option("-o, --output <path>", "Output file path", "flattened-codebase.xml")
|
||||
.action(async (options) => {
|
||||
const inputDir = path.resolve(options.input);
|
||||
const outputPath = path.resolve(options.output);
|
||||
|
||||
let inputDir = path.resolve(options.input);
|
||||
let outputPath = path.resolve(options.output);
|
||||
|
||||
// Detect if user explicitly provided -i/--input or -o/--output
|
||||
const argv = process.argv.slice(2);
|
||||
const userSpecifiedInput = argv.some((a) =>
|
||||
a === "-i" || a === "--input" || a.startsWith("--input=")
|
||||
);
|
||||
const userSpecifiedOutput = argv.some((a) =>
|
||||
a === "-o" || a === "--output" || a.startsWith("--output=")
|
||||
);
|
||||
const noPathArgs = !userSpecifiedInput && !userSpecifiedOutput;
|
||||
|
||||
if (noPathArgs) {
|
||||
const detectedRoot = await findProjectRoot(process.cwd());
|
||||
const suggestedOutput = detectedRoot
|
||||
? path.join(detectedRoot, "flattened-codebase.xml")
|
||||
: path.resolve("flattened-codebase.xml");
|
||||
|
||||
if (detectedRoot) {
|
||||
const useDefaults = await promptYesNo(
|
||||
`Detected project root at "${detectedRoot}". Use it as input and write output to "${suggestedOutput}"?`,
|
||||
true,
|
||||
);
|
||||
if (useDefaults) {
|
||||
inputDir = detectedRoot;
|
||||
outputPath = suggestedOutput;
|
||||
} else {
|
||||
inputDir = await promptPath(
|
||||
"Enter input directory path",
|
||||
process.cwd(),
|
||||
);
|
||||
outputPath = await promptPath(
|
||||
"Enter output file path",
|
||||
path.join(inputDir, "flattened-codebase.xml"),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log("Could not auto-detect a project root.");
|
||||
inputDir = await promptPath(
|
||||
"Enter input directory path",
|
||||
process.cwd(),
|
||||
);
|
||||
outputPath = await promptPath(
|
||||
"Enter output file path",
|
||||
path.join(inputDir, "flattened-codebase.xml"),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.error(
|
||||
"Could not auto-detect a project root and no arguments were provided. Please specify -i/--input and -o/--output.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Ensure output directory exists
|
||||
await fs.ensureDir(path.dirname(outputPath));
|
||||
|
||||
console.log(`Flattening codebase from: ${inputDir}`);
|
||||
console.log(`Output file: ${outputPath}`);
|
||||
|
||||
@@ -598,30 +148,27 @@ program
|
||||
}
|
||||
|
||||
// Import ora dynamically
|
||||
const { default: ora } = await import('ora');
|
||||
const { default: ora } = await import("ora");
|
||||
|
||||
// Start file discovery with spinner
|
||||
const discoverySpinner = ora('🔍 Discovering files...').start();
|
||||
const discoverySpinner = ora("🔍 Discovering files...").start();
|
||||
const files = await discoverFiles(inputDir);
|
||||
const filteredFiles = await filterFiles(files, inputDir);
|
||||
discoverySpinner.succeed(`📁 Found ${filteredFiles.length} files to include`);
|
||||
|
||||
// Write filteredFiles to temp.txt for debugging XML including unneeded files
|
||||
// const tempFilePath = path.join(process.cwd(), 'temp-filtered-files.txt');
|
||||
// await fs.writeFile(
|
||||
// tempFilePath,
|
||||
// filteredFiles.map(file => `${file}\n${path.relative(inputDir, file)}\n---\n`).join('\n')
|
||||
// );
|
||||
// console.log(`📄 Filtered files written to: ${tempFilePath}`);
|
||||
discoverySpinner.succeed(
|
||||
`📁 Found ${filteredFiles.length} files to include`,
|
||||
);
|
||||
|
||||
// Process files with progress tracking
|
||||
console.log('Reading file contents');
|
||||
const processingSpinner = ora('📄 Processing files...').start();
|
||||
const aggregatedContent = await aggregateFileContents(filteredFiles, inputDir, processingSpinner);
|
||||
processingSpinner.succeed(`✅ Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
||||
|
||||
// Log processing results for test validation
|
||||
console.log(`Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
||||
console.log("Reading file contents");
|
||||
const processingSpinner = ora("📄 Processing files...").start();
|
||||
const aggregatedContent = await aggregateFileContents(
|
||||
filteredFiles,
|
||||
inputDir,
|
||||
processingSpinner,
|
||||
);
|
||||
processingSpinner.succeed(
|
||||
`✅ Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`,
|
||||
);
|
||||
if (aggregatedContent.errors.length > 0) {
|
||||
console.log(`Errors: ${aggregatedContent.errors.length}`);
|
||||
}
|
||||
@@ -631,27 +178,34 @@ program
|
||||
}
|
||||
|
||||
// Generate XML output using streaming
|
||||
const xmlSpinner = ora('🔧 Generating XML output...').start();
|
||||
const xmlSpinner = ora("🔧 Generating XML output...").start();
|
||||
await generateXMLOutput(aggregatedContent, outputPath);
|
||||
xmlSpinner.succeed('📝 XML generation completed');
|
||||
xmlSpinner.succeed("📝 XML generation completed");
|
||||
|
||||
// Calculate and display statistics
|
||||
const outputStats = await fs.stat(outputPath);
|
||||
const stats = calculateStatistics(aggregatedContent, outputStats.size);
|
||||
|
||||
// Display completion summary
|
||||
console.log('\n📊 Completion Summary:');
|
||||
console.log(`✅ Successfully processed ${filteredFiles.length} files into ${path.basename(outputPath)}`);
|
||||
console.log("\n📊 Completion Summary:");
|
||||
console.log(
|
||||
`✅ Successfully processed ${filteredFiles.length} files into ${
|
||||
path.basename(outputPath)
|
||||
}`,
|
||||
);
|
||||
console.log(`📁 Output file: ${outputPath}`);
|
||||
console.log(`📏 Total source size: ${stats.totalSize}`);
|
||||
console.log(`📄 Generated XML size: ${stats.xmlSize}`);
|
||||
console.log(`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`);
|
||||
console.log(
|
||||
`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`,
|
||||
);
|
||||
console.log(`🔢 Estimated tokens: ${stats.estimatedTokens}`);
|
||||
console.log(`📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`);
|
||||
|
||||
console.log(
|
||||
`📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('❌ Critical error:', error.message);
|
||||
console.error('An unexpected error occurred.');
|
||||
console.error("❌ Critical error:", error.message);
|
||||
console.error("An unexpected error occurred.");
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
45
tools/flattener/projectRoot.js
Normal file
45
tools/flattener/projectRoot.js
Normal file
@@ -0,0 +1,45 @@
|
||||
const fs = require("fs-extra");
|
||||
const path = require("node:path");
|
||||
|
||||
/**
|
||||
* Attempt to find the project root by walking up from startDir
|
||||
* Looks for common project markers like .git, package.json, pyproject.toml, etc.
|
||||
* @param {string} startDir
|
||||
* @returns {Promise<string|null>} project root directory or null if not found
|
||||
*/
|
||||
async function findProjectRoot(startDir) {
|
||||
try {
|
||||
let dir = path.resolve(startDir);
|
||||
const root = path.parse(dir).root;
|
||||
const markers = [
|
||||
".git",
|
||||
"package.json",
|
||||
"pnpm-workspace.yaml",
|
||||
"yarn.lock",
|
||||
"pnpm-lock.yaml",
|
||||
"pyproject.toml",
|
||||
"requirements.txt",
|
||||
"go.mod",
|
||||
"Cargo.toml",
|
||||
"composer.json",
|
||||
".hg",
|
||||
".svn",
|
||||
];
|
||||
|
||||
while (true) {
|
||||
const exists = await Promise.all(
|
||||
markers.map((m) => fs.pathExists(path.join(dir, m))),
|
||||
);
|
||||
if (exists.some(Boolean)) {
|
||||
return dir;
|
||||
}
|
||||
if (dir === root) break;
|
||||
dir = path.dirname(dir);
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { findProjectRoot };
|
||||
44
tools/flattener/prompts.js
Normal file
44
tools/flattener/prompts.js
Normal file
@@ -0,0 +1,44 @@
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const readline = require("node:readline");
|
||||
const process = require("node:process");
|
||||
|
||||
function expandHome(p) {
|
||||
if (!p) return p;
|
||||
if (p.startsWith("~")) return path.join(os.homedir(), p.slice(1));
|
||||
return p;
|
||||
}
|
||||
|
||||
function createRl() {
|
||||
return readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
}
|
||||
|
||||
function promptQuestion(question) {
|
||||
return new Promise((resolve) => {
|
||||
const rl = createRl();
|
||||
rl.question(question, (answer) => {
|
||||
rl.close();
|
||||
resolve(answer);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function promptYesNo(question, defaultYes = true) {
|
||||
const suffix = defaultYes ? " [Y/n] " : " [y/N] ";
|
||||
const ans = (await promptQuestion(`${question}${suffix}`)).trim().toLowerCase();
|
||||
if (!ans) return defaultYes;
|
||||
if (["y", "yes"].includes(ans)) return true;
|
||||
if (["n", "no"].includes(ans)) return false;
|
||||
return promptYesNo(question, defaultYes);
|
||||
}
|
||||
|
||||
async function promptPath(question, defaultValue) {
|
||||
const prompt = `${question}${defaultValue ? ` (default: ${defaultValue})` : ""}: `;
|
||||
const ans = (await promptQuestion(prompt)).trim();
|
||||
return expandHome(ans || defaultValue);
|
||||
}
|
||||
|
||||
module.exports = { promptYesNo, promptPath, promptQuestion, expandHome };
|
||||
30
tools/flattener/stats.js
Normal file
30
tools/flattener/stats.js
Normal file
@@ -0,0 +1,30 @@
|
||||
function calculateStatistics(aggregatedContent, xmlFileSize) {
|
||||
const { textFiles, binaryFiles, errors } = aggregatedContent;
|
||||
|
||||
const totalTextSize = textFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalBinarySize = binaryFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalSize = totalTextSize + totalBinarySize;
|
||||
|
||||
const totalLines = textFiles.reduce((sum, file) => sum + file.lines, 0);
|
||||
|
||||
const estimatedTokens = Math.ceil(xmlFileSize / 4);
|
||||
|
||||
const formatSize = (bytes) => {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
};
|
||||
|
||||
return {
|
||||
totalFiles: textFiles.length + binaryFiles.length,
|
||||
textFiles: textFiles.length,
|
||||
binaryFiles: binaryFiles.length,
|
||||
errorFiles: errors.length,
|
||||
totalSize: formatSize(totalSize),
|
||||
xmlSize: formatSize(xmlFileSize),
|
||||
totalLines,
|
||||
estimatedTokens: estimatedTokens.toLocaleString(),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { calculateStatistics };
|
||||
86
tools/flattener/xml.js
Normal file
86
tools/flattener/xml.js
Normal file
@@ -0,0 +1,86 @@
|
||||
const fs = require("fs-extra");
|
||||
|
||||
function escapeXml(str) {
|
||||
if (typeof str !== "string") {
|
||||
return String(str);
|
||||
}
|
||||
return str
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function indentFileContent(content) {
|
||||
if (typeof content !== "string") {
|
||||
return String(content);
|
||||
}
|
||||
return content.split("\n").map((line) => ` ${line}`);
|
||||
}
|
||||
|
||||
function generateXMLOutput(aggregatedContent, outputPath) {
|
||||
const { textFiles } = aggregatedContent;
|
||||
const writeStream = fs.createWriteStream(outputPath, { encoding: "utf8" });
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writeStream.on("error", reject);
|
||||
writeStream.on("finish", resolve);
|
||||
|
||||
writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
|
||||
writeStream.write("<files>\n");
|
||||
|
||||
// Sort files by path for deterministic order
|
||||
const filesSorted = [...textFiles].sort((a, b) =>
|
||||
a.path.localeCompare(b.path)
|
||||
);
|
||||
let index = 0;
|
||||
|
||||
const writeNext = () => {
|
||||
if (index >= filesSorted.length) {
|
||||
writeStream.write("</files>\n");
|
||||
writeStream.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const file = filesSorted[index++];
|
||||
const p = escapeXml(file.path);
|
||||
const content = typeof file.content === "string" ? file.content : "";
|
||||
|
||||
if (content.length === 0) {
|
||||
writeStream.write(`\t<file path='${p}'/>\n`);
|
||||
setTimeout(writeNext, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
const needsCdata = content.includes("<") || content.includes("&") ||
|
||||
content.includes("]]>");
|
||||
if (needsCdata) {
|
||||
// Open tag and CDATA on their own line with tab indent; content lines indented with two tabs
|
||||
writeStream.write(`\t<file path='${p}'><![CDATA[\n`);
|
||||
// Safely split any occurrences of "]]>" inside content, trim trailing newlines, indent each line with two tabs
|
||||
const safe = content.replace(/]]>/g, "]]]]><![CDATA[>");
|
||||
const trimmed = safe.replace(/[\r\n]+$/, "");
|
||||
const indented = trimmed.length > 0
|
||||
? trimmed.split("\n").map((line) => `\t\t${line}`).join("\n")
|
||||
: "";
|
||||
writeStream.write(indented);
|
||||
// Close CDATA and attach closing tag directly after the last content line
|
||||
writeStream.write("]]></file>\n");
|
||||
} else {
|
||||
// Write opening tag then newline; indent content with two tabs; attach closing tag directly after last content char
|
||||
writeStream.write(`\t<file path='${p}'>\n`);
|
||||
const trimmed = content.replace(/[\r\n]+$/, "");
|
||||
const indented = trimmed.length > 0
|
||||
? trimmed.split("\n").map((line) => `\t\t${line}`).join("\n")
|
||||
: "";
|
||||
writeStream.write(indented);
|
||||
writeStream.write(`</file>\n`);
|
||||
}
|
||||
|
||||
setTimeout(writeNext, 0);
|
||||
};
|
||||
|
||||
writeNext();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { generateXMLOutput };
|
||||
105
tools/shared/bannerArt.js
Normal file
105
tools/shared/bannerArt.js
Normal file
@@ -0,0 +1,105 @@
|
||||
// ASCII banner art definitions extracted from banners.js to separate art from logic
|
||||
|
||||
const BMAD_TITLE = "BMAD-METHOD";
|
||||
const FLATTENER_TITLE = "FLATTENER";
|
||||
const INSTALLER_TITLE = "INSTALLER";
|
||||
|
||||
// Large ASCII blocks (block-style fonts)
|
||||
const BMAD_LARGE = `
|
||||
██████╗ ███╗ ███╗ █████╗ ██████╗ ███╗ ███╗███████╗████████╗██╗ ██╗ ██████╗ ██████╗
|
||||
██╔══██╗████╗ ████║██╔══██╗██╔══██╗ ████╗ ████║██╔════╝╚══██╔══╝██║ ██║██╔═══██╗██╔══██╗
|
||||
██████╔╝██╔████╔██║███████║██║ ██║█████╗██╔████╔██║█████╗ ██║ ███████║██║ ██║██║ ██║
|
||||
██╔══██╗██║╚██╔╝██║██╔══██║██║ ██║╚════╝██║╚██╔╝██║██╔══╝ ██║ ██╔══██║██║ ██║██║ ██║
|
||||
██████╔╝██║ ╚═╝ ██║██║ ██║██████╔╝ ██║ ╚═╝ ██║███████╗ ██║ ██║ ██║╚██████╔╝██████╔╝
|
||||
╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝╚══════╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═════╝
|
||||
`;
|
||||
|
||||
const FLATTENER_LARGE = `
|
||||
███████╗██╗ █████╗ ████████╗████████╗███████╗███╗ ██╗███████╗██████╗
|
||||
██╔════╝██║ ██╔══██╗╚══██╔══╝╚══██╔══╝██╔════╝████╗ ██║██╔════╝██╔══██╗
|
||||
█████╗ ██║ ███████║ ██║ ██║ █████╗ ██╔██╗ ██║█████╗ ██████╔╝
|
||||
██╔══╝ ██║ ██╔══██║ ██║ ██║ ██╔══╝ ██║╚██╗██║██╔══╝ ██╔══██╗
|
||||
██║ ███████║██║ ██║ ██║ ██║ ███████╗██║ ╚████║███████╗██║ ██║
|
||||
╚═╝ ╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝
|
||||
`;
|
||||
|
||||
const INSTALLER_LARGE = `
|
||||
██╗███╗ ██╗███████╗████████╗ █████╗ ██╗ ██╗ ███████╗██████╗
|
||||
██║████╗ ██║██╔════╝╚══██╔══╝██╔══██╗██║ ██║ ██╔════╝██╔══██╗
|
||||
██║██╔██╗ ██║███████╗ ██║ ███████║██║ ██║ █████╗ ██████╔╝
|
||||
██║██║╚██╗██║╚════██║ ██║ ██╔══██║██║ ██║ ██╔══╝ ██╔══██╗
|
||||
██║██║ ╚████║███████║ ██║ ██║ ██║███████╗███████╗███████╗██║ ██║
|
||||
╚═╝╚═╝ ╚═══╝╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝
|
||||
`;
|
||||
|
||||
// Curated medium/small/tiny variants (fixed art, no runtime scaling)
|
||||
// Medium: bold framed title with heavy fill (high contrast, compact)
|
||||
const BMAD_MEDIUM = `
|
||||
███╗ █╗ █╗ ██╗ ███╗ █╗ █╗███╗█████╗█╗ █╗ ██╗ ███╗
|
||||
█╔═█╗██╗ ██║█╔═█╗█╔═█╗ ██╗ ██║█╔═╝╚═█╔═╝█║ █║█╔═█╗█╔═█╗
|
||||
███╔╝█╔███╔█║████║█║ █║██╗█╔███╔█║██╗ █║ ████║█║ █║█║ █║
|
||||
█╔═█╗█║ █╔╝█║█╔═█║█║ █║╚═╝█║ █╔╝█║█╔╝ █║ █╔═█║█║ █║█║ █║
|
||||
███╔╝█║ ╚╝ █║█║ █║███╔╝ █║ ╚╝ █║███╗ █║ █║ █║╚██╔╝███╔╝
|
||||
╚══╝ ╚╝ ╚╝╚╝ ╚╝╚══╝ ╚╝ ╚╝╚══╝ ╚╝ ╚╝ ╚╝ ╚═╝ ╚══╝
|
||||
`;
|
||||
|
||||
const FLATTENER_MEDIUM = `
|
||||
███╗█╗ ██╗ █████╗█████╗███╗█╗ █╗███╗███╗
|
||||
█╔═╝█║ █╔═█╗╚═█╔═╝╚═█╔═╝█╔═╝██╗ █║█╔═╝█╔═█╗
|
||||
██╗ █║ ████║ █║ █║ ██╗ █╔█╗█║██╗ ███╔╝
|
||||
█╔╝ █║ █╔═█║ █║ █║ █╔╝ █║ ██║█╔╝ █╔═█╗
|
||||
█║ ███║█║ █║ █║ █║ ███╗█║ █║███╗█║ █║
|
||||
╚╝ ╚══╝╚╝ ╚╝ ╚╝ ╚╝ ╚══╝╚╝ ╚╝╚══╝╚╝ ╚╝
|
||||
`;
|
||||
|
||||
const INSTALLER_MEDIUM = `
|
||||
█╗█╗ █╗████╗█████╗ ██╗ █╗ █╗ ███╗███╗
|
||||
█║██╗ █║█╔══╝╚═█╔═╝█╔═█╗█║ █║ █╔═╝█╔═█╗
|
||||
█║█╔█╗█║████╗ █║ ████║█║ █║ ██╗ ███╔╝
|
||||
█║█║ ██║╚══█║ █║ █╔═█║█║ █║ █╔╝ █╔═█╗
|
||||
█║█║ █║████║ █║ █║ █║███╗███╗███╗█║ █║
|
||||
╚╝╚╝ ╚╝╚═══╝ ╚╝ ╚╝ ╚╝╚══╝╚══╝╚══╝╚╝ ╚╝
|
||||
`;
|
||||
|
||||
// Small: rounded box with bold rule
|
||||
// Width: 30 columns total (28 inner)
|
||||
const BMAD_SMALL = `
|
||||
╭──────────────────────────╮
|
||||
│ BMAD-METHOD │
|
||||
╰──────────────────────────╯
|
||||
`;
|
||||
|
||||
const FLATTENER_SMALL = `
|
||||
╭──────────────────────────╮
|
||||
│ FLATTENER │
|
||||
╰──────────────────────────╯
|
||||
`;
|
||||
|
||||
const INSTALLER_SMALL = `
|
||||
╭──────────────────────────╮
|
||||
│ INSTALLER │
|
||||
╰──────────────────────────╯
|
||||
`;
|
||||
|
||||
// Tiny (compact brackets)
|
||||
const BMAD_TINY = `[ BMAD-METHOD ]`;
|
||||
const FLATTENER_TINY = `[ FLATTENER ]`;
|
||||
const INSTALLER_TINY = `[ INSTALLER ]`;
|
||||
|
||||
module.exports = {
|
||||
BMAD_TITLE,
|
||||
FLATTENER_TITLE,
|
||||
INSTALLER_TITLE,
|
||||
BMAD_LARGE,
|
||||
FLATTENER_LARGE,
|
||||
INSTALLER_LARGE,
|
||||
BMAD_MEDIUM,
|
||||
FLATTENER_MEDIUM,
|
||||
INSTALLER_MEDIUM,
|
||||
BMAD_SMALL,
|
||||
FLATTENER_SMALL,
|
||||
INSTALLER_SMALL,
|
||||
BMAD_TINY,
|
||||
FLATTENER_TINY,
|
||||
INSTALLER_TINY,
|
||||
};
|
||||
Reference in New Issue
Block a user