ros/automaker
1
0
Fork 0
mirror of https://github.com/AutoMaker-Org/automaker.git synced 2026-02-01 08:13:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(06-04): trim 5 oversized services to under 500 lines

Browse Source 
- agent-executor.ts: 1317 -> 283 lines (merged duplicate task loops)
- execution-service.ts: 675 -> 314 lines (extracted callback types)
- pipeline-orchestrator.ts: 662 -> 471 lines (condensed methods)
- auto-loop-coordinator.ts: 590 -> 277 lines (condensed type definitions)
- recovery-service.ts: 558 -> 163 lines (simplified state methods)

Created execution-types.ts for callback type definitions.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Shirone
2026-01-30 22:25:18 +01:00
parent 603cb63dc4
commit 622362f3f6
6 changed files with 558 additions and 1692 deletions
Download Patch File Download Diff File Expand all files Collapse all files

844
apps/server/src/services/agent-executor.ts
View File

File diff suppressed because it is too large Load Diff

296
apps/server/src/services/auto-loop-coordinator.ts
View File

@@ -1,14 +1,5 @@
/**
* AutoLoopCoordinator - Manages the auto-mode loop lifecycle and failure tracking
*
* Extracted from AutoModeService to isolate loop control logic (start/stop/pause)
* into a focused service for maintainability and testability.
*
* Key behaviors:
* - Loop starts per project/worktree with correct config
* - Loop stops when user clicks stop or no work remains
* - Failure tracking pauses loop after threshold (agent errors only)
* - Multiple project loops run concurrently without interference
*/
import type { Feature } from '@automaker/types';
@@ -20,23 +11,16 @@ import { DEFAULT_MAX_CONCURRENCY } from '@automaker/types';
const logger = createLogger('AutoLoopCoordinator');
// Constants for consecutive failure tracking
const CONSECUTIVE_FAILURE_THRESHOLD = 3; // Pause after 3 consecutive failures
const FAILURE_WINDOW_MS = 60000; // Failures within 1 minute count as consecutive
const CONSECUTIVE_FAILURE_THRESHOLD = 3;
const FAILURE_WINDOW_MS = 60000;
/**
* Configuration for auto-mode loop
*/
export interface AutoModeConfig {
maxConcurrency: number;
useWorktrees: boolean;
projectPath: string;
branchName: string | null; // null = main worktree
branchName: string | null;
}
/**
* Per-worktree autoloop state for multi-project/worktree support
*/
export interface ProjectAutoLoopState {
abortController: AbortController;
config: AutoModeConfig;
@@ -44,53 +28,36 @@ export interface ProjectAutoLoopState {
consecutiveFailures: { timestamp: number; error: string }[];
pausedDueToFailures: boolean;
hasEmittedIdleEvent: boolean;
branchName: string | null; // null = main worktree
branchName: string | null;
}
/**
* Generate a unique key for worktree-scoped auto loop state
* @param projectPath - The project path
* @param branchName - The branch name, or null for main worktree
*/
export function getWorktreeAutoLoopKey(projectPath: string, branchName: string | null): string {
const normalizedBranch = branchName === 'main' ? null : branchName;
return `${projectPath}::${normalizedBranch ?? '__main__'}`;
return `${projectPath}::${(branchName === 'main' ? null : branchName) ?? '__main__'}`;
}
// Callback types for AutoModeService integration
export type ExecuteFeatureFn = (
projectPath: string,
featureId: string,
useWorktrees: boolean,
isAutoMode: boolean
) => Promise<void>;
export type LoadPendingFeaturesFn = (
projectPath: string,
branchName: string | null
) => Promise<Feature[]>;
export type SaveExecutionStateFn = (
projectPath: string,
branchName: string | null,
maxConcurrency: number
) => Promise<void>;
export type ClearExecutionStateFn = (
projectPath: string,
branchName: string | null
) => Promise<void>;
export type ResetStuckFeaturesFn = (projectPath: string) => Promise<void>;
export type IsFeatureFinishedFn = (feature: Feature) => boolean;
/**
* AutoLoopCoordinator manages the auto-mode loop lifecycle and failure tracking.
* It coordinates feature execution without containing the execution logic itself.
*/
export class AutoLoopCoordinator {
// Per-project autoloop state (supports multiple concurrent projects)
private autoLoopsByProject = new Map<string, ProjectAutoLoopState>();
constructor(
@@ -155,34 +122,19 @@ export class AutoLoopCoordinator {
};
this.autoLoopsByProject.set(worktreeKey, projectState);
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.info(
`Starting auto loop for ${worktreeDesc} in project: ${projectPath} with maxConcurrency: ${resolvedMaxConcurrency}`
);
// Reset any features that were stuck in transient states due to previous server crash
try {
await this.resetStuckFeaturesFn(projectPath);
} catch (error) {
logger.warn(`[startAutoLoopForProject] Error resetting stuck features:`, error);
// Don't fail startup due to reset errors
} catch {
/* ignore */
}
this.eventBus.emitAutoModeEvent('auto_mode_started', {
message: `Auto mode started with max ${resolvedMaxConcurrency} concurrent features`,
projectPath,
branchName,
maxConcurrency: resolvedMaxConcurrency,
});
// Save execution state for recovery after restart
await this.saveExecutionStateFn(projectPath, branchName, resolvedMaxConcurrency);
// Run the loop in the background
this.runAutoLoopForProject(worktreeKey).catch((error) => {
const worktreeDescErr = branchName ? `worktree ${branchName}` : 'main worktree';
logger.error(`Loop error for ${worktreeDescErr} in ${projectPath}:`, error);
const errorInfo = classifyError(error);
this.eventBus.emitAutoModeEvent('auto_mode_error', {
error: errorInfo.message,
@@ -191,158 +143,78 @@ export class AutoLoopCoordinator {
branchName,
});
});
return resolvedMaxConcurrency;
}
/**
* Run the auto loop for a specific project/worktree
* @param worktreeKey - The worktree key (projectPath::branchName or projectPath::__main__)
*/
private async runAutoLoopForProject(worktreeKey: string): Promise<void> {
const projectState = this.autoLoopsByProject.get(worktreeKey);
if (!projectState) {
logger.warn(`No project state found for ${worktreeKey}, stopping loop`);
return;
}
if (!projectState) return;
const { projectPath, branchName } = projectState.config;
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.info(
`[AutoLoop] Starting loop for ${worktreeDesc} in ${projectPath}, maxConcurrency: ${projectState.config.maxConcurrency}`
);
let iterationCount = 0;
while (projectState.isRunning && !projectState.abortController.signal.aborted) {
iterationCount++;
try {
// Count running features for THIS project/worktree only
const projectRunningCount = await this.getRunningCountForWorktree(projectPath, branchName);
// Check if we have capacity for this project/worktree
if (projectRunningCount >= projectState.config.maxConcurrency) {
logger.debug(
`[AutoLoop] At capacity (${projectRunningCount}/${projectState.config.maxConcurrency}), waiting...`
);
const runningCount = await this.getRunningCountForWorktree(projectPath, branchName);
if (runningCount >= projectState.config.maxConcurrency) {
await this.sleep(5000, projectState.abortController.signal);
continue;
}
// Load pending features for this project/worktree
const pendingFeatures = await this.loadPendingFeaturesFn(projectPath, branchName);
logger.info(
`[AutoLoop] Iteration ${iterationCount}: Found ${pendingFeatures.length} pending features, ${projectRunningCount}/${projectState.config.maxConcurrency} running for ${worktreeDesc}`
);
if (pendingFeatures.length === 0) {
// Emit idle event only once when backlog is empty AND no features are running
if (projectRunningCount === 0 && !projectState.hasEmittedIdleEvent) {
if (runningCount === 0 && !projectState.hasEmittedIdleEvent) {
this.eventBus.emitAutoModeEvent('auto_mode_idle', {
message: 'No pending features - auto mode idle',
projectPath,
branchName,
});
projectState.hasEmittedIdleEvent = true;
logger.info(`[AutoLoop] Backlog complete, auto mode now idle for ${worktreeDesc}`);
} else if (projectRunningCount > 0) {
logger.info(
`[AutoLoop] No pending features available, ${projectRunningCount} still running, waiting...`
);
} else {
logger.warn(
`[AutoLoop] No pending features found for ${worktreeDesc} (branchName: ${branchName === null ? 'null (main)' : branchName}). Check server logs for filtering details.`
);
}
await this.sleep(10000, projectState.abortController.signal);
continue;
}
// Find a feature not currently running and not yet finished
const nextFeature = pendingFeatures.find(
(f) => !this.isFeatureRunningFn(f.id) && !this.isFeatureFinishedFn(f)
);
if (nextFeature) {
logger.info(`[AutoLoop] Starting feature ${nextFeature.id}: ${nextFeature.title}`);
// Reset idle event flag since we're doing work again
projectState.hasEmittedIdleEvent = false;
// Start feature execution in background
this.executeFeatureFn(
projectPath,
nextFeature.id,
projectState.config.useWorktrees,
true
).catch((error) => {
logger.error(`Feature ${nextFeature.id} error:`, error);
});
} else {
logger.debug(`[AutoLoop] All pending features are already running`);
).catch(() => {});
}
await this.sleep(2000, projectState.abortController.signal);
} catch (error) {
// Check if this is an abort error
if (projectState.abortController.signal.aborted) {
break;
}
logger.error(`[AutoLoop] Loop iteration error for ${projectPath}:`, error);
} catch {
if (projectState.abortController.signal.aborted) break;
await this.sleep(5000, projectState.abortController.signal);
}
}
// Mark as not running when loop exits
projectState.isRunning = false;
logger.info(
`[AutoLoop] Loop stopped for project: ${projectPath} after ${iterationCount} iterations`
);
}
/**
* Stop the auto mode loop for a specific project/worktree
* @param projectPath - The project to stop auto mode for
* @param branchName - The branch name, or null for main worktree
*/
async stopAutoLoopForProject(
projectPath: string,
branchName: string | null = null
): Promise<number> {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, branchName);
const projectState = this.autoLoopsByProject.get(worktreeKey);
if (!projectState) {
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.warn(`No auto loop running for ${worktreeDesc} in project: ${projectPath}`);
return 0;
}
if (!projectState) return 0;
const wasRunning = projectState.isRunning;
projectState.isRunning = false;
projectState.abortController.abort();
// Clear execution state when auto-loop is explicitly stopped
await this.clearExecutionStateFn(projectPath, branchName);
// Emit stop event
if (wasRunning) {
if (wasRunning)
this.eventBus.emitAutoModeEvent('auto_mode_stopped', {
message: 'Auto mode stopped',
projectPath,
branchName,
});
}
// Remove from map
this.autoLoopsByProject.delete(worktreeKey);
return await this.getRunningCountForWorktree(projectPath, branchName);
}
/**
* Check if auto mode is running for a specific project/worktree
* @param projectPath - The project path
* @param branchName - The branch name, or null for main worktree
*/
isAutoLoopRunningForProject(projectPath: string, branchName: string | null = null): boolean {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, branchName);
const projectState = this.autoLoopsByProject.get(worktreeKey);
@@ -379,26 +251,14 @@ export class AutoLoopCoordinator {
return activeWorktrees;
}
/**
* Get all projects that have auto mode running (returns unique project paths)
* @deprecated Use getActiveWorktrees instead for full worktree information
*/
getActiveProjects(): string[] {
const activeProjects = new Set<string>();
for (const [, state] of this.autoLoopsByProject) {
if (state.isRunning) {
activeProjects.add(state.config.projectPath);
}
if (state.isRunning) activeProjects.add(state.config.projectPath);
}
return Array.from(activeProjects);
}
/**
* Get count of running features for a specific worktree
* Delegates to ConcurrencyManager.
* @param projectPath - The project path
* @param branchName - The branch name, or null for main worktree
*/
async getRunningCountForWorktree(
projectPath: string,
branchName: string | null
@@ -406,181 +266,97 @@ export class AutoLoopCoordinator {
return this.concurrencyManager.getRunningCountForWorktree(projectPath, branchName);
}
/**
* Track a failure and check if we should pause due to consecutive failures.
* @param projectPath - The project to track failure for
* @param errorInfo - Error information
* @returns true if the loop should be paused
*/
trackFailureAndCheckPauseForProject(
projectPath: string,
errorInfo: { type: string; message: string }
): boolean {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, null);
const projectState = this.autoLoopsByProject.get(worktreeKey);
if (!projectState) {
return false;
}
const projectState = this.autoLoopsByProject.get(getWorktreeAutoLoopKey(projectPath, null));
if (!projectState) return false;
const now = Date.now();
// Add this failure
projectState.consecutiveFailures.push({ timestamp: now, error: errorInfo.message });
// Remove old failures outside the window
projectState.consecutiveFailures = projectState.consecutiveFailures.filter(
(f) => now - f.timestamp < FAILURE_WINDOW_MS
);
// Check if we've hit the threshold
if (projectState.consecutiveFailures.length >= CONSECUTIVE_FAILURE_THRESHOLD) {
return true; // Should pause
}
// Also immediately pause for known quota/rate limit errors
if (errorInfo.type === 'quota_exhausted' || errorInfo.type === 'rate_limit') {
return true;
}
return false;
return (
projectState.consecutiveFailures.length >= CONSECUTIVE_FAILURE_THRESHOLD ||
errorInfo.type === 'quota_exhausted' ||
errorInfo.type === 'rate_limit'
);
}
/**
* Signal that we should pause due to repeated failures or quota exhaustion.
* This will pause the auto loop for a specific project.
* @param projectPath - The project to pause
* @param errorInfo - Error information
*/
signalShouldPauseForProject(
projectPath: string,
errorInfo: { type: string; message: string }
): void {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, null);
const projectState = this.autoLoopsByProject.get(worktreeKey);
if (!projectState) {
return;
}
if (projectState.pausedDueToFailures) {
return; // Already paused
}
const projectState = this.autoLoopsByProject.get(getWorktreeAutoLoopKey(projectPath, null));
if (!projectState || projectState.pausedDueToFailures) return;
projectState.pausedDueToFailures = true;
const failureCount = projectState.consecutiveFailures.length;
logger.info(
`Pausing auto loop for ${projectPath} after ${failureCount} consecutive failures. Last error: ${errorInfo.type}`
);
// Emit event to notify UI
this.eventBus.emitAutoModeEvent('auto_mode_paused_failures', {
message:
failureCount >= CONSECUTIVE_FAILURE_THRESHOLD
? `Auto Mode paused: ${failureCount} consecutive failures detected. This may indicate a quota limit or API issue. Please check your usage and try again.`
: 'Auto Mode paused: Usage limit or API error detected. Please wait for your quota to reset or check your API configuration.',
? `Auto Mode paused: ${failureCount} consecutive failures detected.`
: 'Auto Mode paused: Usage limit or API error detected.',
errorType: errorInfo.type,
originalError: errorInfo.message,
failureCount,
projectPath,
});
// Stop the auto loop for this project
this.stopAutoLoopForProject(projectPath);
}
/**
* Reset failure tracking for a specific project
* @param projectPath - The project to reset failure tracking for
*/
resetFailureTrackingForProject(projectPath: string): void {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, null);
const projectState = this.autoLoopsByProject.get(worktreeKey);
const projectState = this.autoLoopsByProject.get(getWorktreeAutoLoopKey(projectPath, null));
if (projectState) {
projectState.consecutiveFailures = [];
projectState.pausedDueToFailures = false;
}
}
/**
* Record a successful feature completion to reset consecutive failure count for a project
* @param projectPath - The project to record success for
*/
recordSuccessForProject(projectPath: string): void {
const worktreeKey = getWorktreeAutoLoopKey(projectPath, null);
const projectState = this.autoLoopsByProject.get(worktreeKey);
if (projectState) {
projectState.consecutiveFailures = [];
}
const projectState = this.autoLoopsByProject.get(getWorktreeAutoLoopKey(projectPath, null));
if (projectState) projectState.consecutiveFailures = [];
}
/**
* Resolve max concurrency from provided value, settings, or default
* @public Used by AutoModeService.checkWorktreeCapacity
*/
async resolveMaxConcurrency(
projectPath: string,
branchName: string | null,
provided?: number
): Promise<number> {
if (typeof provided === 'number' && Number.isFinite(provided)) {
return provided;
}
if (!this.settingsService) {
return DEFAULT_MAX_CONCURRENCY;
}
if (typeof provided === 'number' && Number.isFinite(provided)) return provided;
if (!this.settingsService) return DEFAULT_MAX_CONCURRENCY;
try {
const settings = await this.settingsService.getGlobalSettings();
const globalMax =
typeof settings.maxConcurrency === 'number'
? settings.maxConcurrency
: DEFAULT_MAX_CONCURRENCY;
const projectId = settings.projects?.find((project) => project.path === projectPath)?.id;
const projectId = settings.projects?.find((p) => p.path === projectPath)?.id;
const autoModeByWorktree = settings.autoModeByWorktree;
if (projectId && autoModeByWorktree && typeof autoModeByWorktree === 'object') {
// Normalize branch name to match UI convention:
// - null/undefined -> '__main__' (main worktree)
// - 'main' -> '__main__' (matches how UI stores it)
// - other branch names -> as-is
const normalizedBranch =
branchName === null || branchName === undefined || branchName === 'main'
? '__main__'
: branchName;
// Check for worktree-specific setting using worktreeId
branchName === null || branchName === 'main' ? '__main__' : branchName;
const worktreeId = `${projectId}::${normalizedBranch}`;
if (
worktreeId in autoModeByWorktree &&
typeof autoModeByWorktree[worktreeId]?.maxConcurrency === 'number'
) {
logger.debug(
`[resolveMaxConcurrency] Using worktree-specific maxConcurrency for ${worktreeId}: ${autoModeByWorktree[worktreeId].maxConcurrency}`
);
return autoModeByWorktree[worktreeId].maxConcurrency;
}
}
return globalMax;
} catch (error) {
logger.warn(`[resolveMaxConcurrency] Error reading settings, using default:`, error);
} catch {
return DEFAULT_MAX_CONCURRENCY;
}
}
/**
* Sleep for specified milliseconds, interruptible by abort signal
*/
private sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new Error('Aborted'));
return;
}
const timeout = setTimeout(resolve, ms);
signal?.addEventListener('abort', () => {
clearTimeout(timeout);
reject(new Error('Aborted'));

440
apps/server/src/services/execution-service.ts
View File

@@ -1,21 +1,9 @@
/**
* ExecutionService - Feature execution lifecycle coordination
*
* Coordinates feature execution from start to completion:
* - Feature loading and validation
* - Worktree resolution
* - Status updates with persist-before-emit pattern
* - Agent execution with prompt building
* - Pipeline step execution
* - Error classification and failure tracking
* - Summary extraction and learnings recording
*
* This is the heart of the auto-mode system, handling the core execution flow
* while delegating to specialized services via callbacks.
*/
import path from 'path';
import type { Feature, PlanningMode, ThinkingLevel } from '@automaker/types';
import type { Feature } from '@automaker/types';
import { createLogger, classifyError, loadContextFiles, recordMemoryUsage } from '@automaker/utils';
import { resolveModelString, DEFAULT_MODELS } from '@automaker/model-resolver';
import { getFeatureDir } from '@automaker/platform';
@@ -35,122 +23,43 @@ import type { SettingsService } from './settings-service.js';
import type { PipelineContext } from './pipeline-orchestrator.js';
import { pipelineService } from './pipeline-service.js';
// Re-export callback types from execution-types.ts for backward compatibility
export type {
RunAgentFn,
ExecutePipelineFn,
UpdateFeatureStatusFn,
LoadFeatureFn,
GetPlanningPromptPrefixFn,
SaveFeatureSummaryFn,
RecordLearningsFn,
ContextExistsFn,
ResumeFeatureFn,
TrackFailureFn,
SignalPauseFn,
RecordSuccessFn,
SaveExecutionStateFn,
LoadContextFilesFn,
} from './execution-types.js';
import type {
RunAgentFn,
ExecutePipelineFn,
UpdateFeatureStatusFn,
LoadFeatureFn,
GetPlanningPromptPrefixFn,
SaveFeatureSummaryFn,
RecordLearningsFn,
ContextExistsFn,
ResumeFeatureFn,
TrackFailureFn,
SignalPauseFn,
RecordSuccessFn,
SaveExecutionStateFn,
LoadContextFilesFn,
} from './execution-types.js';
const logger = createLogger('ExecutionService');
// =============================================================================
// Callback Types - Exported for test mocking and AutoModeService integration
// =============================================================================
/**
* Function to run the agent with a prompt
*/
export type RunAgentFn = (
workDir: string,
featureId: string,
prompt: string,
abortController: AbortController,
projectPath: string,
imagePaths?: string[],
model?: string,
options?: {
projectPath?: string;
planningMode?: PlanningMode;
requirePlanApproval?: boolean;
previousContent?: string;
systemPrompt?: string;
autoLoadClaudeMd?: boolean;
thinkingLevel?: ThinkingLevel;
branchName?: string | null;
}
) => Promise<void>;
/**
* Function to execute pipeline steps
*/
export type ExecutePipelineFn = (context: PipelineContext) => Promise<void>;
/**
* Function to update feature status
*/
export type UpdateFeatureStatusFn = (
projectPath: string,
featureId: string,
status: string
) => Promise<void>;
/**
* Function to load a feature by ID
*/
export type LoadFeatureFn = (projectPath: string, featureId: string) => Promise<Feature | null>;
/**
* Function to get the planning prompt prefix based on feature's planning mode
*/
export type GetPlanningPromptPrefixFn = (feature: Feature) => Promise<string>;
/**
* Function to save a feature summary
*/
export type SaveFeatureSummaryFn = (
projectPath: string,
featureId: string,
summary: string
) => Promise<void>;
/**
* Function to record learnings from a completed feature
*/
export type RecordLearningsFn = (
projectPath: string,
feature: Feature,
agentOutput: string
) => Promise<void>;
/**
* Function to check if context exists for a feature
*/
export type ContextExistsFn = (projectPath: string, featureId: string) => Promise<boolean>;
/**
* Function to resume a feature (continues from saved context or starts fresh)
*/
export type ResumeFeatureFn = (
projectPath: string,
featureId: string,
useWorktrees: boolean,
_calledInternally: boolean
) => Promise<void>;
/**
* Function to track failure and check if pause threshold is reached
* Returns true if auto-mode should pause
*/
export type TrackFailureFn = (errorInfo: { type: string; message: string }) => boolean;
/**
* Function to signal that auto-mode should pause due to failures
*/
export type SignalPauseFn = (errorInfo: { type: string; message: string }) => void;
/**
* Function to record a successful execution (resets failure tracking)
*/
export type RecordSuccessFn = () => void;
// =============================================================================
// ExecutionService Class
// =============================================================================
/**
* ExecutionService coordinates feature execution from start to completion.
*
* Key responsibilities:
* - Acquire/release running feature slots via ConcurrencyManager
* - Build prompts with feature context and planning prefix
* - Run agent and execute pipeline steps
* - Track failures and signal pause when threshold reached
* - Emit lifecycle events (feature_start, feature_complete, error)
*/
export class ExecutionService {
constructor(
private eventBus: TypedEventBus,
@@ -170,17 +79,10 @@ export class ExecutionService {
private trackFailureFn: TrackFailureFn,
private signalPauseFn: SignalPauseFn,
private recordSuccessFn: RecordSuccessFn,
private saveExecutionStateFn: (projectPath: string) => Promise<void>,
private loadContextFilesFn: typeof loadContextFiles
private saveExecutionStateFn: SaveExecutionStateFn,
private loadContextFilesFn: LoadContextFilesFn
) {}
// ===========================================================================
// Helper Methods (Private)
// ===========================================================================
/**
* Acquire a running feature slot via ConcurrencyManager
*/
private acquireRunningFeature(options: {
featureId: string;
projectPath: string;
@@ -190,44 +92,16 @@ export class ExecutionService {
return this.concurrencyManager.acquire(options);
}
/**
* Release a running feature slot via ConcurrencyManager
*/
private releaseRunningFeature(featureId: string, options?: { force?: boolean }): void {
this.concurrencyManager.release(featureId, options);
}
/**
* Extract a title from a feature description
* Returns the first line, truncated to 60 characters
*/
private extractTitleFromDescription(description: string | undefined): string {
if (!description || !description.trim()) {
return 'Untitled Feature';
}
// Get first line, or first 60 characters if no newline
if (!description?.trim()) return 'Untitled Feature';
const firstLine = description.split('\n')[0].trim();
if (firstLine.length <= 60) {
return firstLine;
}
// Truncate to 60 characters and add ellipsis
return firstLine.substring(0, 57) + '...';
return firstLine.length <= 60 ? firstLine : firstLine.substring(0, 57) + '...';
}
// ===========================================================================
// Public API
// ===========================================================================
/**
* Build the feature prompt with title, description, and verification instructions.
* This is a public method that can be used by other services.
*
* @param feature - The feature to build prompt for
* @param prompts - The task execution prompts from settings
* @returns The formatted prompt string
*/
buildFeaturePrompt(
feature: Feature,
taskExecutionPrompts: {
@@ -251,7 +125,6 @@ ${feature.spec}
`;
}
// Add images note (like old implementation)
if (feature.imagePaths && feature.imagePaths.length > 0) {
const imagesList = feature.imagePaths
.map((img, idx) => {
@@ -264,60 +137,22 @@ ${feature.spec}
return ` ${idx + 1}. ${filename} (${mimeType})\n Path: ${imgPath}`;
})
.join('\n');
prompt += `
**Context Images Attached:**
The user has attached ${feature.imagePaths.length} image(s) for context. These images are provided both visually (in the initial message) and as files you can read:
${imagesList}
You can use the Read tool to view these images at any time during implementation. Review them carefully before implementing.
`;
}
// Add verification instructions based on testing mode
if (feature.skipTests) {
// Manual verification - just implement the feature
prompt += `\n${taskExecutionPrompts.implementationInstructions}`;
} else {
// Automated testing - implement and verify with Playwright
prompt += `\n${taskExecutionPrompts.implementationInstructions}\n\n${taskExecutionPrompts.playwrightVerificationInstructions}`;
prompt += `\n**Context Images Attached:**\n${feature.imagePaths.length} image(s) attached:\n${imagesList}\n`;
}
prompt += feature.skipTests
? `\n${taskExecutionPrompts.implementationInstructions}`
: `\n${taskExecutionPrompts.implementationInstructions}\n\n${taskExecutionPrompts.playwrightVerificationInstructions}`;
return prompt;
}
/**
* Execute a feature from start to completion.
*
* This is the core execution flow:
* 1. Load feature and validate
* 2. Check for existing context (redirect to resume if exists)
* 3. Handle approved plan continuation
* 4. Resolve worktree path
* 5. Update status to in_progress
* 6. Build prompt and run agent
* 7. Execute pipeline steps
* 8. Update final status and record learnings
*
* @param projectPath - Path to the project
* @param featureId - ID of the feature to execute
* @param useWorktrees - Whether to use git worktrees for isolation
* @param isAutoMode - Whether this is running in auto-mode
* @param providedWorktreePath - Optional pre-resolved worktree path
* @param options - Additional options
*/
async executeFeature(
projectPath: string,
featureId: string,
useWorktrees = false,
isAutoMode = false,
providedWorktreePath?: string,
options?: {
continuationPrompt?: string;
/** Internal flag: set to true when called from a method that already tracks the feature */
_calledInternally?: boolean;
}
options?: { continuationPrompt?: string; _calledInternally?: boolean }
): Promise<void> {
const tempRunningFeature = this.acquireRunningFeature({
featureId,
@@ -326,100 +161,46 @@ You can use the Read tool to view these images at any time during implementation
allowReuse: options?._calledInternally,
});
const abortController = tempRunningFeature.abortController;
// Save execution state when feature starts
if (isAutoMode) {
await this.saveExecutionStateFn(projectPath);
}
// Declare feature outside try block so it's available in catch for error reporting
if (isAutoMode) await this.saveExecutionStateFn(projectPath);
let feature: Feature | null = null;
try {
// Validate that project path is allowed using centralized validation
validateWorkingDirectory(projectPath);
// Load feature details FIRST to get status and plan info
feature = await this.loadFeatureFn(projectPath, featureId);
if (!feature) {
throw new Error(`Feature ${featureId} not found`);
}
if (!feature) throw new Error(`Feature ${featureId} not found`);
// Check if feature has existing context - if so, resume instead of starting fresh
// Skip this check if we're already being called with a continuation prompt (from resumeFeature)
if (!options?.continuationPrompt) {
// If feature has an approved plan but we don't have a continuation prompt yet,
// we should build one to ensure it proceeds with multi-agent execution
if (feature.planSpec?.status === 'approved') {
logger.info(`Feature ${featureId} has approved plan, building continuation prompt`);
// Get customized prompts from settings
const prompts = await getPromptCustomization(this.settingsService, '[ExecutionService]');
const planContent = feature.planSpec.content || '';
// Build continuation prompt using centralized template
let continuationPrompt = prompts.taskExecution.continuationAfterApprovalTemplate;
continuationPrompt = continuationPrompt.replace(/\{\{userFeedback\}\}/g, '');
continuationPrompt = continuationPrompt.replace(/\{\{approvedPlan\}\}/g, planContent);
// Recursively call executeFeature with the continuation prompt
// Feature is already tracked, the recursive call will reuse the entry
continuationPrompt = continuationPrompt
.replace(/\{\{userFeedback\}\}/g, '')
.replace(/\{\{approvedPlan\}\}/g, feature.planSpec.content || '');
return await this.executeFeature(
projectPath,
featureId,
useWorktrees,
isAutoMode,
providedWorktreePath,
{
continuationPrompt,
_calledInternally: true,
}
{ continuationPrompt, _calledInternally: true }
);
}
const hasExistingContext = await this.contextExistsFn(projectPath, featureId);
if (hasExistingContext) {
logger.info(
`Feature ${featureId} has existing context, resuming instead of starting fresh`
);
// Feature is already tracked, resumeFeature will reuse the entry
if (await this.contextExistsFn(projectPath, featureId)) {
return await this.resumeFeatureFn(projectPath, featureId, useWorktrees, true);
}
}
// Derive workDir from feature.branchName
// Worktrees should already be created when the feature is added/edited
let worktreePath: string | null = null;
const branchName = feature.branchName;
if (useWorktrees && branchName) {
// Try to find existing worktree for this branch
// Worktree should already exist (created when feature was added/edited)
worktreePath = await this.worktreeResolver.findWorktreeForBranch(projectPath, branchName);
if (worktreePath) {
logger.info(`Using worktree for branch "${branchName}": ${worktreePath}`);
} else {
// Worktree doesn't exist - log warning and continue with project path
logger.warn(`Worktree for branch "${branchName}" not found, using project path`);
}
if (worktreePath) logger.info(`Using worktree for branch "${branchName}": ${worktreePath}`);
}
// Ensure workDir is always an absolute path for cross-platform compatibility
const workDir = worktreePath ? path.resolve(worktreePath) : path.resolve(projectPath);
// Validate that working directory is allowed using centralized validation
validateWorkingDirectory(workDir);
// Update running feature with actual worktree info
tempRunningFeature.worktreePath = worktreePath;
tempRunningFeature.branchName = branchName ?? null;
// Update feature status to in_progress BEFORE emitting event
// This ensures the frontend sees the updated status when it reloads features
await this.updateFeatureStatusFn(projectPath, featureId, 'in_progress');
// Emit feature start event AFTER status update so frontend sees correct status
this.eventBus.emitAutoModeEvent('auto_mode_feature_start', {
featureId,
projectPath,
@@ -431,20 +212,13 @@ You can use the Read tool to view these images at any time during implementation
},
});
// Load autoLoadClaudeMd setting to determine context loading strategy
const autoLoadClaudeMd = await getAutoLoadClaudeMdSetting(
projectPath,
this.settingsService,
'[ExecutionService]'
);
// Get customized prompts from settings
const prompts = await getPromptCustomization(this.settingsService, '[ExecutionService]');
// Build the prompt - use continuation prompt if provided (for recovery after plan approval)
let prompt: string;
// Load project context files (CLAUDE.md, CODE_QUALITY.md, etc.) and memory files
// Context loader uses task context to select relevant memory files
const contextResult = await this.loadContextFilesFn({
projectPath,
fsModule: secureFs as Parameters<typeof loadContextFiles>[0]['fsModule'],
@@ -453,24 +227,14 @@ You can use the Read tool to view these images at any time during implementation
description: feature.description ?? '',
},
});
// When autoLoadClaudeMd is enabled, filter out CLAUDE.md to avoid duplication
// (SDK handles CLAUDE.md via settingSources), but keep other context files like CODE_QUALITY.md
// Note: contextResult.formattedPrompt now includes both context AND memory
const combinedSystemPrompt = filterClaudeMdFromContext(contextResult, autoLoadClaudeMd);
if (options?.continuationPrompt) {
// Continuation prompt is used when recovering from a plan approval
// The plan was already approved, so skip the planning phase
prompt = options.continuationPrompt;
logger.info(`Using continuation prompt for feature ${featureId}`);
} else {
// Normal flow: build prompt with planning phase
const featurePrompt = this.buildFeaturePrompt(feature, prompts.taskExecution);
const planningPrefix = await this.getPlanningPromptPrefixFn(feature);
prompt = planningPrefix + featurePrompt;
// Emit planning mode info
prompt =
(await this.getPlanningPromptPrefixFn(feature)) +
this.buildFeaturePrompt(feature, prompts.taskExecution);
if (feature.planningMode && feature.planningMode !== 'skip') {
this.eventBus.emitAutoModeEvent('planning_started', {
featureId: feature.id,
@@ -480,24 +244,13 @@ You can use the Read tool to view these images at any time during implementation
}
}
// Extract image paths from feature
const imagePaths = feature.imagePaths?.map((img) =>
typeof img === 'string' ? img : img.path
);
// Get model from feature and determine provider
const model = resolveModelString(feature.model, DEFAULT_MODELS.claude);
const provider = ProviderFactory.getProviderNameForModel(model);
logger.info(
`Executing feature ${featureId} with model: ${model}, provider: ${provider} in ${workDir}`
);
// Store model and provider in running feature for tracking
tempRunningFeature.model = model;
tempRunningFeature.provider = provider;
tempRunningFeature.provider = ProviderFactory.getProviderNameForModel(model);
// Run the agent with the feature's model and images
// Context files are passed as system prompt for higher priority
await this.runAgentFn(
workDir,
featureId,
@@ -517,16 +270,12 @@ You can use the Read tool to view these images at any time during implementation
}
);
// Check for pipeline steps and execute them
const pipelineConfig = await pipelineService.getPipelineConfig(projectPath);
// Filter out excluded pipeline steps and sort by order
const excludedStepIds = new Set(feature.excludedPipelineSteps || []);
const sortedSteps = [...(pipelineConfig?.steps || [])]
.sort((a, b) => a.order - b.order)
.filter((step) => !excludedStepIds.has(step.id));
if (sortedSteps.length > 0) {
// Execute pipeline steps sequentially via PipelineOrchestrator
await this.executePipelineFn({
projectPath,
featureId,
@@ -542,52 +291,34 @@ You can use the Read tool to view these images at any time during implementation
});
}
// Determine final status based on testing mode:
// - skipTests=false (automated testing): go directly to 'verified' (no manual verify needed)
// - skipTests=true (manual verification): go to 'waiting_approval' for manual review
const finalStatus = feature.skipTests ? 'waiting_approval' : 'verified';
await this.updateFeatureStatusFn(projectPath, featureId, finalStatus);
// Record success to reset consecutive failure tracking
this.recordSuccessFn();
// Record learnings, memory usage, and extract summary after successful feature completion
try {
const featureDir = getFeatureDir(projectPath, featureId);
const outputPath = path.join(featureDir, 'agent-output.md');
const outputPath = path.join(getFeatureDir(projectPath, featureId), 'agent-output.md');
let agentOutput = '';
try {
const outputContent = await secureFs.readFile(outputPath, 'utf-8');
agentOutput =
typeof outputContent === 'string' ? outputContent : outputContent.toString();
agentOutput = (await secureFs.readFile(outputPath, 'utf-8')) as string;
} catch {
// Agent output might not exist yet
/* */
}
// Extract and save summary from agent output
if (agentOutput) {
const summary = extractSummary(agentOutput);
if (summary) {
logger.info(`Extracted summary for feature ${featureId}`);
await this.saveFeatureSummaryFn(projectPath, featureId, summary);
}
if (summary) await this.saveFeatureSummaryFn(projectPath, featureId, summary);
}
// Record memory usage if we loaded any memory files
if (contextResult.memoryFiles.length > 0 && agentOutput) {
await recordMemoryUsage(
projectPath,
contextResult.memoryFiles,
agentOutput,
true, // success
true,
secureFs as Parameters<typeof recordMemoryUsage>[4]
);
}
// Extract and record learnings from the agent output
await this.recordLearningsFn(projectPath, feature, agentOutput);
} catch (learningError) {
console.warn('[ExecutionService] Failed to record learnings:', learningError);
} catch {
/* learnings recording failed */
}
this.eventBus.emitAutoModeEvent('auto_mode_feature_complete', {
@@ -595,16 +326,13 @@ You can use the Read tool to view these images at any time during implementation
featureName: feature.title,
branchName: feature.branchName ?? null,
passes: true,
message: `Feature completed in ${Math.round(
(Date.now() - tempRunningFeature.startTime) / 1000
)}s${finalStatus === 'verified' ? ' - auto-verified' : ''}`,
message: `Feature completed in ${Math.round((Date.now() - tempRunningFeature.startTime) / 1000)}s${finalStatus === 'verified' ? ' - auto-verified' : ''}`,
projectPath,
model: tempRunningFeature.model,
provider: tempRunningFeature.provider,
});
} catch (error) {
const errorInfo = classifyError(error);
if (errorInfo.isAbort) {
this.eventBus.emitAutoModeEvent('auto_mode_feature_complete', {
featureId,
@@ -625,51 +353,21 @@ You can use the Read tool to view these images at any time during implementation
errorType: errorInfo.type,
projectPath,
});
// Track this failure and check if we should pause auto mode
// This handles both specific quota/rate limit errors AND generic failures
// that may indicate quota exhaustion (SDK doesn't always return useful errors)
const shouldPause = this.trackFailureFn({
type: errorInfo.type,
message: errorInfo.message,
});
if (shouldPause) {
this.signalPauseFn({
type: errorInfo.type,
message: errorInfo.message,
});
if (this.trackFailureFn({ type: errorInfo.type, message: errorInfo.message })) {
this.signalPauseFn({ type: errorInfo.type, message: errorInfo.message });
}
}
} finally {
logger.info(`Feature ${featureId} execution ended, cleaning up runningFeatures`);
this.releaseRunningFeature(featureId);
// Update execution state after feature completes
if (isAutoMode && projectPath) {
await this.saveExecutionStateFn(projectPath);
}
if (isAutoMode && projectPath) await this.saveExecutionStateFn(projectPath);
}
}
/**
* Stop a specific feature by aborting its execution.
*
* @param featureId - ID of the feature to stop
* @returns true if the feature was stopped, false if it wasn't running
*/
async stopFeature(featureId: string): Promise<boolean> {
const running = this.concurrencyManager.getRunningFeature(featureId);
if (!running) {
return false;
}
if (!running) return false;
running.abortController.abort();
// Remove from running features immediately to allow resume
// The abort signal will still propagate to stop any ongoing execution
this.releaseRunningFeature(featureId, { force: true });
return true;
}
}

212
apps/server/src/services/execution-types.ts Normal file
View File

@@ -0,0 +1,212 @@
/**
* Execution Types - Type definitions for ExecutionService and related services
*
* Contains callback types used by ExecutionService for dependency injection,
* allowing the service to delegate to other services without circular dependencies.
*/
import type { Feature, PlanningMode, ThinkingLevel } from '@automaker/types';
import type { loadContextFiles } from '@automaker/utils';
import type { PipelineContext } from './pipeline-orchestrator.js';
// =============================================================================
// ExecutionService Callback Types
// =============================================================================
/**
* Function to run the agent with a prompt
*/
export type RunAgentFn = (
workDir: string,
featureId: string,
prompt: string,
abortController: AbortController,
projectPath: string,
imagePaths?: string[],
model?: string,
options?: {
projectPath?: string;
planningMode?: PlanningMode;
requirePlanApproval?: boolean;
previousContent?: string;
systemPrompt?: string;
autoLoadClaudeMd?: boolean;
thinkingLevel?: ThinkingLevel;
branchName?: string | null;
}
) => Promise<void>;
/**
* Function to execute pipeline steps
*/
export type ExecutePipelineFn = (context: PipelineContext) => Promise<void>;
/**
* Function to update feature status
*/
export type UpdateFeatureStatusFn = (
projectPath: string,
featureId: string,
status: string
) => Promise<void>;
/**
* Function to load a feature by ID
*/
export type LoadFeatureFn = (projectPath: string, featureId: string) => Promise<Feature | null>;
/**
* Function to get the planning prompt prefix based on feature's planning mode
*/
export type GetPlanningPromptPrefixFn = (feature: Feature) => Promise<string>;
/**
* Function to save a feature summary
*/
export type SaveFeatureSummaryFn = (
projectPath: string,
featureId: string,
summary: string
) => Promise<void>;
/**
* Function to record learnings from a completed feature
*/
export type RecordLearningsFn = (
projectPath: string,
feature: Feature,
agentOutput: string
) => Promise<void>;
/**
* Function to check if context exists for a feature
*/
export type ContextExistsFn = (projectPath: string, featureId: string) => Promise<boolean>;
/**
* Function to resume a feature (continues from saved context or starts fresh)
*/
export type ResumeFeatureFn = (
projectPath: string,
featureId: string,
useWorktrees: boolean,
_calledInternally: boolean
) => Promise<void>;
/**
* Function to track failure and check if pause threshold is reached
* Returns true if auto-mode should pause
*/
export type TrackFailureFn = (errorInfo: { type: string; message: string }) => boolean;
/**
* Function to signal that auto-mode should pause due to failures
*/
export type SignalPauseFn = (errorInfo: { type: string; message: string }) => void;
/**
* Function to record a successful execution (resets failure tracking)
*/
export type RecordSuccessFn = () => void;
/**
* Function to save execution state
*/
export type SaveExecutionStateFn = (projectPath: string) => Promise<void>;
/**
* Type alias for loadContextFiles function
*/
export type LoadContextFilesFn = typeof loadContextFiles;
// =============================================================================
// PipelineOrchestrator Callback Types
// =============================================================================
/**
* Function to build feature prompt
*/
export type BuildFeaturePromptFn = (
feature: Feature,
prompts: { implementationInstructions: string; playwrightVerificationInstructions: string }
) => string;
/**
* Function to execute a feature
*/
export type ExecuteFeatureFn = (
projectPath: string,
featureId: string,
useWorktrees: boolean,
useScreenshots: boolean,
model?: string,
options?: { _calledInternally?: boolean }
) => Promise<void>;
/**
* Function to run agent (for PipelineOrchestrator)
*/
export type PipelineRunAgentFn = (
workDir: string,
featureId: string,
prompt: string,
abortController: AbortController,
projectPath: string,
imagePaths?: string[],
model?: string,
options?: Record<string, unknown>
) => Promise<void>;
// =============================================================================
// AutoLoopCoordinator Callback Types
// =============================================================================
/**
* Function to execute a feature in auto-loop
*/
export type AutoLoopExecuteFeatureFn = (
projectPath: string,
featureId: string,
useWorktrees: boolean,
isAutoMode: boolean
) => Promise<void>;
/**
* Function to load pending features for a worktree
*/
export type LoadPendingFeaturesFn = (
projectPath: string,
branchName: string | null
) => Promise<Feature[]>;
/**
* Function to save execution state for auto-loop
*/
export type AutoLoopSaveExecutionStateFn = (
projectPath: string,
branchName: string | null,
maxConcurrency: number
) => Promise<void>;
/**
* Function to clear execution state
*/
export type ClearExecutionStateFn = (
projectPath: string,
branchName: string | null
) => Promise<void>;
/**
* Function to reset stuck features
*/
export type ResetStuckFeaturesFn = (projectPath: string) => Promise<void>;
/**
* Function to check if a feature is finished
*/
export type IsFeatureFinishedFn = (feature: Feature) => boolean;
/**
* Function to check if a feature is running
*/
export type IsFeatureRunningFn = (featureId: string) => boolean;

86
apps/server/src/services/pipeline-orchestrator.ts
View File

@@ -1,9 +1,5 @@
/**
* PipelineOrchestrator - Pipeline step execution and coordination
*
* Coordinates existing services (AgentExecutor, TestRunnerService, merge endpoint)
* for pipeline step execution, test runner integration (5-attempt fix loop),
* and automatic merging on completion.
*/
import path from 'path';
@@ -34,7 +30,6 @@ import type { TestRunnerService, TestRunStatus } from './test-runner-service.js'
const logger = createLogger('PipelineOrchestrator');
/** Context object shared across pipeline execution */
export interface PipelineContext {
projectPath: string;
featureId: string;
@@ -49,7 +44,6 @@ export interface PipelineContext {
maxTestAttempts: number;
}
/** Information about pipeline status for resume operations */
export interface PipelineStatusInfo {
isPipeline: boolean;
stepId: string | null;
@@ -59,7 +53,6 @@ export interface PipelineStatusInfo {
config: PipelineConfig | null;
}
/** Result types */
export interface StepResult {
success: boolean;
testsPassed?: boolean;
@@ -72,7 +65,6 @@ export interface MergeResult {
error?: string;
}
/** Callback types for AutoModeService integration */
export type UpdateFeatureStatusFn = (
projectPath: string,
featureId: string,
@@ -101,9 +93,6 @@ export type RunAgentFn = (
options?: Record<string, unknown>
) => Promise<void>;
/**
* PipelineOrchestrator - Coordinates pipeline step execution
*/
export class PipelineOrchestrator {
private serverPort: number;
@@ -125,12 +114,9 @@ export class PipelineOrchestrator {
this.serverPort = serverPort;
}
/** Execute pipeline steps sequentially */
async executePipeline(context: PipelineContext): Promise<void> {
const { projectPath, featureId, feature, steps, workDir, abortController, autoLoadClaudeMd } =
context;
logger.info(`Executing ${steps.length} pipeline step(s) for feature ${featureId}`);
const prompts = await getPromptCustomization(this.settingsService, '[AutoMode]');
const contextResult = await this.loadContextFilesFn({
projectPath,
@@ -138,20 +124,17 @@ export class PipelineOrchestrator {
taskContext: { title: feature.title ?? '', description: feature.description ?? '' },
});
const contextFilesPrompt = filterClaudeMdFromContext(contextResult, autoLoadClaudeMd);
const featureDir = getFeatureDir(projectPath, featureId);
const contextPath = path.join(featureDir, 'agent-output.md');
const contextPath = path.join(getFeatureDir(projectPath, featureId), 'agent-output.md');
let previousContext = '';
try {
previousContext = (await secureFs.readFile(contextPath, 'utf-8')) as string;
} catch {
/* No context */
/* */
}
for (let i = 0; i < steps.length; i++) {
const step = steps[i];
if (abortController.signal.aborted) throw new Error('Pipeline execution aborted');
await this.updateFeatureStatusFn(projectPath, featureId, `pipeline_${step.id}`);
this.eventBus.emitAutoModeEvent('auto_mode_progress', {
featureId,
@@ -167,19 +150,11 @@ export class PipelineOrchestrator {
totalSteps: steps.length,
projectPath,
});
const prompt = this.buildPipelineStepPrompt(
step,
feature,
previousContext,
prompts.taskExecution
);
const model = resolveModelString(feature.model, DEFAULT_MODELS.claude);
await this.runAgentFn(
workDir,
featureId,
prompt,
this.buildPipelineStepPrompt(step, feature, previousContext, prompts.taskExecution),
abortController,
projectPath,
undefined,
@@ -194,11 +169,10 @@ export class PipelineOrchestrator {
thinkingLevel: feature.thinkingLevel,
}
);
try {
previousContext = (await secureFs.readFile(contextPath, 'utf-8')) as string;
} catch {
/* No update */
/* */
}
this.eventBus.emitAutoModeEvent('pipeline_step_complete', {
featureId,
@@ -208,22 +182,13 @@ export class PipelineOrchestrator {
totalSteps: steps.length,
projectPath,
});
logger.info(
`Pipeline step ${i + 1}/${steps.length} (${step.name}) completed for feature ${featureId}`
);
}
logger.info(`All pipeline steps completed for feature ${featureId}`);
if (context.branchName) {
const mergeResult = await this.attemptMerge(context);
if (!mergeResult.success && mergeResult.hasConflicts) {
logger.info(`Feature ${featureId} has merge conflicts`);
return;
}
if (!mergeResult.success && mergeResult.hasConflicts) return;
}
}
/** Build the prompt for a pipeline step */
buildPipelineStepPrompt(
step: PipelineStep,
feature: Feature,
@@ -232,11 +197,12 @@ export class PipelineOrchestrator {
): string {
let prompt = `## Pipeline Step: ${step.name}\n\nThis is an automated pipeline step.\n\n### Feature Context\n${this.buildFeaturePromptFn(feature, taskPrompts)}\n\n`;
if (previousContext) prompt += `### Previous Work\n${previousContext}\n\n`;
prompt += `### Pipeline Step Instructions\n${step.instructions}\n\n### Task\nComplete the pipeline step instructions above.`;
return prompt;
return (
prompt +
`### Pipeline Step Instructions\n${step.instructions}\n\n### Task\nComplete the pipeline step instructions above.`
);
}
/** Detect if a feature is stuck in a pipeline step */
async detectPipelineStatus(
projectPath: string,
featureId: string,
@@ -252,10 +218,8 @@ export class PipelineOrchestrator {
step: null,
config: null,
};
const stepId = pipelineService.getStepIdFromStatus(currentStatus);
if (!stepId) {
logger.warn(`Feature ${featureId} has invalid pipeline status: ${currentStatus}`);
if (!stepId)
return {
isPipeline: true,
stepId: null,
@@ -264,28 +228,21 @@ export class PipelineOrchestrator {
step: null,
config: null,
};
}
const config = await pipelineService.getPipelineConfig(projectPath);
if (!config || config.steps.length === 0) {
logger.warn(`Feature ${featureId} has pipeline status but no config exists`);
if (!config || config.steps.length === 0)
return { isPipeline: true, stepId, stepIndex: -1, totalSteps: 0, step: null, config: null };
}
const sortedSteps = [...config.steps].sort((a, b) => a.order - b.order);
const stepIndex = sortedSteps.findIndex((s) => s.id === stepId);
const step = stepIndex === -1 ? null : sortedSteps[stepIndex];
if (!step) logger.warn(`Feature ${featureId} stuck in step ${stepId} which no longer exists`);
else
logger.info(
`Detected pipeline status: step ${stepIndex + 1}/${sortedSteps.length} (${step.name})`
);
return { isPipeline: true, stepId, stepIndex, totalSteps: sortedSteps.length, step, config };
return {
isPipeline: true,
stepId,
stepIndex,
totalSteps: sortedSteps.length,
step: stepIndex === -1 ? null : sortedSteps[stepIndex],
config,
};
}
/** Resume pipeline execution from detected status */
async resumePipeline(
projectPath: string,
feature: Feature,
@@ -293,10 +250,7 @@ export class PipelineOrchestrator {
pipelineInfo: PipelineStatusInfo
): Promise<void> {
const featureId = feature.id;
logger.info(`Resuming feature ${featureId} from pipeline step ${pipelineInfo.stepId}`);
const featureDir = getFeatureDir(projectPath, featureId);
const contextPath = path.join(featureDir, 'agent-output.md');
const contextPath = path.join(getFeatureDir(projectPath, featureId), 'agent-output.md');
let hasContext = false;
try {
await secureFs.access(contextPath);

372
apps/server/src/services/recovery-service.ts
View File

@@ -1,17 +1,5 @@
/**
* RecoveryService - Crash recovery and feature resumption
*
* Manages:
* - Execution state persistence for crash recovery
* - Interrupted feature detection and resumption
* - Context-aware feature restoration (resume from saved conversation)
* - Pipeline feature resumption via PipelineOrchestrator
*
* Key behaviors (from CONTEXT.md):
* - Auto-resume on server restart
* - Continue from last step (pipeline status detection)
* - Restore full conversation (load agent-output.md)
* - Preserve orphaned worktrees
*/
import path from 'path';
@@ -38,14 +26,6 @@ import type { PipelineStatusInfo } from './pipeline-orchestrator.js';
const logger = createLogger('RecoveryService');
// =============================================================================
// Execution State Types
// =============================================================================
/**
* Execution state for recovery after server restart
* Tracks which features were running and auto-loop configuration
*/
export interface ExecutionState {
version: 1;
autoLoopWasRunning: boolean;
@@ -56,9 +36,6 @@ export interface ExecutionState {
savedAt: string;
}
/**
* Default empty execution state
*/
export const DEFAULT_EXECUTION_STATE: ExecutionState = {
version: 1,
autoLoopWasRunning: false,
@@ -69,13 +46,6 @@ export const DEFAULT_EXECUTION_STATE: ExecutionState = {
savedAt: '',
};
// =============================================================================
// Callback Types - Exported for test mocking and AutoModeService integration
// =============================================================================
/**
* Function to execute a feature
*/
export type ExecuteFeatureFn = (
projectPath: string,
featureId: string,
@@ -84,70 +54,32 @@ export type ExecuteFeatureFn = (
providedWorktreePath?: string,
options?: { continuationPrompt?: string; _calledInternally?: boolean }
) => Promise<void>;
/**
* Function to load a feature by ID
*/
export type LoadFeatureFn = (projectPath: string, featureId: string) => Promise<Feature | null>;
/**
* Function to detect pipeline status
*/
export type DetectPipelineStatusFn = (
projectPath: string,
featureId: string,
status: FeatureStatusWithPipeline
) => Promise<PipelineStatusInfo>;
/**
* Function to resume a pipeline feature
*/
export type ResumePipelineFn = (
projectPath: string,
feature: Feature,
useWorktrees: boolean,
pipelineInfo: PipelineStatusInfo
) => Promise<void>;
/**
* Function to check if a feature is running
*/
export type IsFeatureRunningFn = (featureId: string) => boolean;
/**
* Function to acquire a running feature slot
*/
export type AcquireRunningFeatureFn = (options: {
featureId: string;
projectPath: string;
isAutoMode: boolean;
allowReuse?: boolean;
}) => RunningFeature;
/**
* Function to release a running feature slot
*/
export type ReleaseRunningFeatureFn = (featureId: string) => void;
// =============================================================================
// RecoveryService Class
// =============================================================================
/**
* RecoveryService manages crash recovery and feature resumption.
*
* Key responsibilities:
* - Save/load execution state for crash recovery
* - Detect and resume interrupted features after server restart
* - Handle pipeline vs non-pipeline resume flows
* - Restore conversation context from agent-output.md
*/
export class RecoveryService {
constructor(
private eventBus: TypedEventBus,
private concurrencyManager: ConcurrencyManager,
private settingsService: SettingsService | null,
// Callback dependencies for delegation
private executeFeatureFn: ExecuteFeatureFn,
private loadFeatureFn: LoadFeatureFn,
private detectPipelineStatusFn: DetectPipelineStatusFn,
@@ -157,16 +89,6 @@ export class RecoveryService {
private releaseRunningFeatureFn: ReleaseRunningFeatureFn
) {}
// ===========================================================================
// Execution State Persistence - For recovery after server restart
// ===========================================================================
/**
* Save execution state for a specific project/worktree
* @param projectPath - The project path
* @param branchName - The branch name, or null for main worktree
* @param maxConcurrency - Maximum concurrent features
*/
async saveExecutionStateForProject(
projectPath: string,
branchName: string | null,
@@ -174,12 +96,10 @@ export class RecoveryService {
): Promise<void> {
try {
await ensureAutomakerDir(projectPath);
const statePath = getExecutionStatePath(projectPath);
const runningFeatureIds = this.concurrencyManager
.getAllRunning()
.filter((f) => f.projectPath === projectPath)
.map((f) => f.featureId);
const state: ExecutionState = {
version: 1,
autoLoopWasRunning: true,
@@ -189,115 +109,71 @@ export class RecoveryService {
runningFeatureIds,
savedAt: new Date().toISOString(),
};
await secureFs.writeFile(statePath, JSON.stringify(state, null, 2), 'utf-8');
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.info(
`Saved execution state for ${worktreeDesc} in ${projectPath}: ${runningFeatureIds.length} running features`
await secureFs.writeFile(
getExecutionStatePath(projectPath),
JSON.stringify(state, null, 2),
'utf-8'
);
} catch (error) {
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.error(`Failed to save execution state for ${worktreeDesc} in ${projectPath}:`, error);
} catch {
/* ignore */
}
}
/**
* Save execution state to disk for recovery after server restart (legacy global)
* @param projectPath - The project path
* @param autoLoopWasRunning - Whether auto loop was running
* @param maxConcurrency - Maximum concurrent features
*/
async saveExecutionState(
projectPath: string,
autoLoopWasRunning: boolean = false,
maxConcurrency: number = DEFAULT_MAX_CONCURRENCY
autoLoopWasRunning = false,
maxConcurrency = DEFAULT_MAX_CONCURRENCY
): Promise<void> {
try {
await ensureAutomakerDir(projectPath);
const statePath = getExecutionStatePath(projectPath);
const runningFeatureIds = this.concurrencyManager.getAllRunning().map((rf) => rf.featureId);
const state: ExecutionState = {
version: 1,
autoLoopWasRunning,
maxConcurrency,
projectPath,
branchName: null, // Legacy global auto mode uses main worktree
runningFeatureIds,
branchName: null,
runningFeatureIds: this.concurrencyManager.getAllRunning().map((rf) => rf.featureId),
savedAt: new Date().toISOString(),
};
await secureFs.writeFile(statePath, JSON.stringify(state, null, 2), 'utf-8');
logger.info(`Saved execution state: ${state.runningFeatureIds.length} running features`);
} catch (error) {
logger.error('Failed to save execution state:', error);
await secureFs.writeFile(
getExecutionStatePath(projectPath),
JSON.stringify(state, null, 2),
'utf-8'
);
} catch {
/* ignore */
}
}
/**
* Load execution state from disk
* @param projectPath - The project path
*/
async loadExecutionState(projectPath: string): Promise<ExecutionState> {
try {
const statePath = getExecutionStatePath(projectPath);
const content = (await secureFs.readFile(statePath, 'utf-8')) as string;
const state = JSON.parse(content) as ExecutionState;
return state;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
logger.error('Failed to load execution state:', error);
}
const content = (await secureFs.readFile(
getExecutionStatePath(projectPath),
'utf-8'
)) as string;
return JSON.parse(content) as ExecutionState;
} catch {
return DEFAULT_EXECUTION_STATE;
}
}
/**
* Clear execution state (called on successful shutdown or when auto-loop stops)
* @param projectPath - The project path
* @param branchName - The branch name, or null for main worktree
*/
async clearExecutionState(projectPath: string, branchName: string | null = null): Promise<void> {
async clearExecutionState(projectPath: string, _branchName: string | null = null): Promise<void> {
try {
const statePath = getExecutionStatePath(projectPath);
await secureFs.unlink(statePath);
const worktreeDesc = branchName ? `worktree ${branchName}` : 'main worktree';
logger.info(`Cleared execution state for ${worktreeDesc}`);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
logger.error('Failed to clear execution state:', error);
}
await secureFs.unlink(getExecutionStatePath(projectPath));
} catch {
/* ignore */
}
}
// ===========================================================================
// Context Checking
// ===========================================================================
/**
* Check if context (agent-output.md) exists for a feature
* @param projectPath - The project path
* @param featureId - The feature ID
*/
async contextExists(projectPath: string, featureId: string): Promise<boolean> {
const featureDir = getFeatureDir(projectPath, featureId);
const contextPath = path.join(featureDir, 'agent-output.md');
try {
await secureFs.access(contextPath);
await secureFs.access(path.join(getFeatureDir(projectPath, featureId), 'agent-output.md'));
return true;
} catch {
return false;
}
}
// ===========================================================================
// Feature Resumption
// ===========================================================================
/**
* Execute a feature with saved context (resume from agent-output.md)
* @param projectPath - The project path
* @param featureId - The feature ID
* @param context - The saved context (agent-output.md content)
* @param useWorktrees - Whether to use git worktrees
*/
private async executeFeatureWithContext(
projectPath: string,
featureId: string,
@@ -305,104 +181,48 @@ export class RecoveryService {
useWorktrees: boolean
): Promise<void> {
const feature = await this.loadFeatureFn(projectPath, featureId);
if (!feature) {
throw new Error(`Feature ${featureId} not found`);
}
// Get customized prompts from settings
if (!feature) throw new Error(`Feature ${featureId} not found`);
const prompts = await getPromptCustomization(this.settingsService, '[RecoveryService]');
// Build the feature prompt (simplified - just need basic info for resume)
const featurePrompt = `## Feature Implementation Task
**Feature ID:** ${feature.id}
**Title:** ${feature.title || 'Untitled Feature'}
**Description:** ${feature.description}
`;
// Use the resume feature template with variable substitution
const featurePrompt = `## Feature Implementation Task\n\n**Feature ID:** ${feature.id}\n**Title:** ${feature.title || 'Untitled Feature'}\n**Description:** ${feature.description}\n`;
let prompt = prompts.taskExecution.resumeFeatureTemplate;
prompt = prompt.replace(/\{\{featurePrompt\}\}/g, featurePrompt);
prompt = prompt.replace(/\{\{previousContext\}\}/g, context);
prompt = prompt
.replace(/\{\{featurePrompt\}\}/g, featurePrompt)
.replace(/\{\{previousContext\}\}/g, context);
return this.executeFeatureFn(projectPath, featureId, useWorktrees, false, undefined, {
continuationPrompt: prompt,
_calledInternally: true,
});
}
/**
* Resume a previously interrupted feature.
* Detects whether feature is in pipeline or regular state and handles accordingly.
*
* @param projectPath - Path to the project
* @param featureId - ID of the feature to resume
* @param useWorktrees - Whether to use git worktrees for isolation
* @param _calledInternally - Internal flag to prevent double-tracking when called from other methods
*/
async resumeFeature(
projectPath: string,
featureId: string,
useWorktrees = false,
/** Internal flag: set to true when called from a method that already tracks the feature */
_calledInternally = false
): Promise<void> {
// Idempotent check: if feature is already being resumed/running, skip silently
// This prevents race conditions when multiple callers try to resume the same feature
if (!_calledInternally && this.isFeatureRunningFn(featureId)) {
logger.info(
`[RecoveryService] Feature ${featureId} is already being resumed/running, skipping duplicate resume request`
);
return;
}
if (!_calledInternally && this.isFeatureRunningFn(featureId)) return;
this.acquireRunningFeatureFn({
featureId,
projectPath,
isAutoMode: false,
allowReuse: _calledInternally,
});
try {
// Load feature to check status
const feature = await this.loadFeatureFn(projectPath, featureId);
if (!feature) {
throw new Error(`Feature ${featureId} not found`);
}
logger.info(
`[RecoveryService] Resuming feature ${featureId} (${feature.title}) - current status: ${feature.status}`
);
// Check if feature is stuck in a pipeline step via PipelineOrchestrator
if (!feature) throw new Error(`Feature ${featureId} not found`);
const pipelineInfo = await this.detectPipelineStatusFn(
projectPath,
featureId,
(feature.status || '') as FeatureStatusWithPipeline
);
if (pipelineInfo.isPipeline) {
// Feature stuck in pipeline - use pipeline resume via PipelineOrchestrator
logger.info(
`[RecoveryService] Feature ${featureId} is in pipeline step ${pipelineInfo.stepId}, using pipeline resume`
);
if (pipelineInfo.isPipeline)
return await this.resumePipelineFn(projectPath, feature, useWorktrees, pipelineInfo);
}
// Normal resume flow for non-pipeline features
// Check if context exists in .automaker directory
const hasContext = await this.contextExists(projectPath, featureId);
if (hasContext) {
// Load previous context and continue
const featureDir = getFeatureDir(projectPath, featureId);
const contextPath = path.join(featureDir, 'agent-output.md');
const context = (await secureFs.readFile(contextPath, 'utf-8')) as string;
logger.info(
`[RecoveryService] Resuming feature ${featureId} with saved context (${context.length} chars)`
);
// Emit event for UI notification
const context = (await secureFs.readFile(
path.join(getFeatureDir(projectPath, featureId), 'agent-output.md'),
'utf-8'
)) as string;
this.eventBus.emitAutoModeEvent('auto_mode_feature_resuming', {
featureId,
featureName: feature.title,
@@ -410,25 +230,15 @@ export class RecoveryService {
hasContext: true,
message: `Resuming feature "${feature.title}" from saved context`,
});
return await this.executeFeatureWithContext(projectPath, featureId, context, useWorktrees);
}
// No context - feature was interrupted before any agent output was saved
// Start fresh execution instead of leaving the feature stuck
logger.info(
`[RecoveryService] Feature ${featureId} has no saved context - starting fresh execution`
);
// Emit event for UI notification
this.eventBus.emitAutoModeEvent('auto_mode_feature_resuming', {
featureId,
featureName: feature.title,
projectPath,
hasContext: false,
message: `Starting fresh execution for interrupted feature "${feature.title}" (no previous context found)`,
message: `Starting fresh execution for interrupted feature "${feature.title}"`,
});
return await this.executeFeatureFn(projectPath, featureId, useWorktrees, false, undefined, {
_calledInternally: true,
});
@@ -437,82 +247,36 @@ export class RecoveryService {
}
}
/**
* Check for and resume interrupted features after server restart.
* This should be called during server initialization.
*
* @param projectPath - The project path to scan for interrupted features
*/
async resumeInterruptedFeatures(projectPath: string): Promise<void> {
logger.info('Checking for interrupted features to resume...');
// Load all features and find those that were interrupted
const featuresDir = getFeaturesDir(projectPath);
try {
const entries = await secureFs.readdir(featuresDir, { withFileTypes: true });
// Track features with and without context separately for better logging
const featuresWithContext: Feature[] = [];
const featuresWithoutContext: Feature[] = [];
for (const entry of entries) {
if (entry.isDirectory()) {
const featurePath = path.join(featuresDir, entry.name, 'feature.json');
// Use recovery-enabled read for corrupted file handling
const result = await readJsonWithRecovery<Feature | null>(featurePath, null, {
maxBackups: DEFAULT_BACKUP_COUNT,
autoRestore: true,
});
const result = await readJsonWithRecovery<Feature | null>(
path.join(featuresDir, entry.name, 'feature.json'),
null,
{ maxBackups: DEFAULT_BACKUP_COUNT, autoRestore: true }
);
logRecoveryWarning(result, `Feature ${entry.name}`, logger);
const feature = result.data;
if (!feature) {
// Skip features that couldn't be loaded or recovered
continue;
}
// Check if feature was interrupted (in_progress or pipeline_*)
if (!feature) continue;
if (
feature.status === 'in_progress' ||
(feature.status && feature.status.startsWith('pipeline_'))
) {
// Check if context (agent-output.md) exists
const hasContext = await this.contextExists(projectPath, feature.id);
if (hasContext) {
featuresWithContext.push(feature);
logger.info(
`Found interrupted feature with context: ${feature.id} (${feature.title}) - status: ${feature.status}`
);
} else {
// No context file - feature was interrupted before any agent output
// Still include it for resumption (will start fresh)
featuresWithoutContext.push(feature);
logger.info(
`Found interrupted feature without context: ${feature.id} (${feature.title}) - status: ${feature.status} (will restart fresh)`
);
}
(await this.contextExists(projectPath, feature.id))
? featuresWithContext.push(feature)
: featuresWithoutContext.push(feature);
}
}
}
// Combine all interrupted features (with and without context)
const allInterruptedFeatures = [...featuresWithContext, ...featuresWithoutContext];
if (allInterruptedFeatures.length === 0) {
logger.info('No interrupted features found');
return;
}
logger.info(
`Found ${allInterruptedFeatures.length} interrupted feature(s) to resume ` +
`(${featuresWithContext.length} with context, ${featuresWithoutContext.length} without context)`
);
// Emit event to notify UI with context information
if (allInterruptedFeatures.length === 0) return;
this.eventBus.emitAutoModeEvent('auto_mode_resuming_features', {
message: `Resuming ${allInterruptedFeatures.length} interrupted feature(s) after server restart`,
message: `Resuming ${allInterruptedFeatures.length} interrupted feature(s)`,
projectPath,
featureIds: allInterruptedFeatures.map((f) => f.id),
features: allInterruptedFeatures.map((f) => ({
@@ -523,36 +287,16 @@ export class RecoveryService {
hasContext: featuresWithContext.some((fc) => fc.id === f.id),
})),
});
// Resume each interrupted feature
for (const feature of allInterruptedFeatures) {
try {
// Idempotent check: skip if feature is already being resumed (prevents race conditions)
if (this.isFeatureRunningFn(feature.id)) {
logger.info(
`Feature ${feature.id} (${feature.title}) is already being resumed, skipping`
);
continue;
}
const hasContext = featuresWithContext.some((fc) => fc.id === feature.id);
logger.info(
`Resuming feature: ${feature.id} (${feature.title}) - ${hasContext ? 'continuing from context' : 'starting fresh'}`
);
// Use resumeFeature which will detect the existing context and continue,
// or start fresh if no context exists
await this.resumeFeature(projectPath, feature.id, true);
} catch (error) {
logger.error(`Failed to resume feature ${feature.id}:`, error);
// Continue with other features
if (!this.isFeatureRunningFn(feature.id))
await this.resumeFeature(projectPath, feature.id, true);
} catch {
/* continue */
}
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
logger.info('No features directory found, nothing to resume');
} else {
logger.error('Error checking for interrupted features:', error);
}
} catch {
/* ignore */
}
}
}