diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index a78e9e83..54fcc247 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -386,6 +386,30 @@ eventHookService.initialize(events, settingsService, eventHistoryService, featur await agentService.initialize(); logger.info('Agent service initialized'); + // Reconcile feature states on startup + // After any type of restart (clean, forced, crash), features may be stuck in + // transient states (in_progress, interrupted, pipeline_*) that don't match reality. + // Reconcile them back to resting states before the UI is served. + try { + const settings = await settingsService.getGlobalSettings(); + if (settings.projects && settings.projects.length > 0) { + let totalReconciled = 0; + for (const project of settings.projects) { + const count = await autoModeService.reconcileFeatureStates(project.path); + totalReconciled += count; + } + if (totalReconciled > 0) { + logger.info( + `[STARTUP] Reconciled ${totalReconciled} feature(s) across ${settings.projects.length} project(s)` + ); + } else { + logger.info('[STARTUP] Feature state reconciliation complete - no stale states found'); + } + } + } catch (err) { + logger.warn('[STARTUP] Failed to reconcile feature states:', err); + } + // Bootstrap Codex model cache in background (don't block server startup) void codexModelCacheService.getModels().catch((err) => { logger.error('Failed to bootstrap Codex model cache:', err); diff --git a/apps/server/src/routes/auto-mode/index.ts b/apps/server/src/routes/auto-mode/index.ts index a0c998d6..016447d7 100644 --- a/apps/server/src/routes/auto-mode/index.ts +++ b/apps/server/src/routes/auto-mode/index.ts @@ -21,6 +21,7 @@ import { createFollowUpFeatureHandler } from './routes/follow-up-feature.js'; import { createCommitFeatureHandler } from './routes/commit-feature.js'; import { createApprovePlanHandler } from './routes/approve-plan.js'; import { createResumeInterruptedHandler } from './routes/resume-interrupted.js'; +import { createReconcileHandler } from './routes/reconcile.js'; /** * Create auto-mode routes. @@ -81,6 +82,11 @@ export function createAutoModeRoutes(autoModeService: AutoModeServiceCompat): Ro validatePathParams('projectPath'), createResumeInterruptedHandler(autoModeService) ); + router.post( + '/reconcile', + validatePathParams('projectPath'), + createReconcileHandler(autoModeService) + ); return router; } diff --git a/apps/server/src/routes/auto-mode/routes/reconcile.ts b/apps/server/src/routes/auto-mode/routes/reconcile.ts new file mode 100644 index 00000000..96109051 --- /dev/null +++ b/apps/server/src/routes/auto-mode/routes/reconcile.ts @@ -0,0 +1,53 @@ +/** + * Reconcile Feature States Handler + * + * On-demand endpoint to reconcile all feature states for a project. + * Resets features stuck in transient states (in_progress, interrupted, pipeline_*) + * back to resting states (ready/backlog) and emits events to update the UI. + * + * This is useful when: + * - The UI reconnects after a server restart + * - A client detects stale feature states + * - An admin wants to force-reset stuck features + */ + +import type { Request, Response } from 'express'; +import { createLogger } from '@automaker/utils'; +import type { AutoModeServiceCompat } from '../../../services/auto-mode/index.js'; + +const logger = createLogger('ReconcileFeatures'); + +interface ReconcileRequest { + projectPath: string; +} + +export function createReconcileHandler(autoModeService: AutoModeServiceCompat) { + return async (req: Request, res: Response): Promise => { + const { projectPath } = req.body as ReconcileRequest; + + if (!projectPath) { + res.status(400).json({ error: 'Project path is required' }); + return; + } + + logger.info(`Reconciling feature states for ${projectPath}`); + + try { + const reconciledCount = await autoModeService.reconcileFeatureStates(projectPath); + + res.json({ + success: true, + reconciledCount, + message: + reconciledCount > 0 + ? `Reconciled ${reconciledCount} feature(s)` + : 'No features needed reconciliation', + }); + } catch (error) { + logger.error('Error reconciling feature states:', error); + res.status(500).json({ + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; +} diff --git a/apps/server/src/services/auto-mode/compat.ts b/apps/server/src/services/auto-mode/compat.ts index 2c713c01..97fe19e8 100644 --- a/apps/server/src/services/auto-mode/compat.ts +++ b/apps/server/src/services/auto-mode/compat.ts @@ -88,6 +88,10 @@ export class AutoModeServiceCompat { return this.globalService.markAllRunningFeaturesInterrupted(reason); } + async reconcileFeatureStates(projectPath: string): Promise { + return this.globalService.reconcileFeatureStates(projectPath); + } + // =========================================================================== // PER-PROJECT OPERATIONS (delegated to facades) // =========================================================================== diff --git a/apps/server/src/services/auto-mode/global-service.ts b/apps/server/src/services/auto-mode/global-service.ts index 0e0e7e52..90576f8c 100644 --- a/apps/server/src/services/auto-mode/global-service.ts +++ b/apps/server/src/services/auto-mode/global-service.ts @@ -205,4 +205,21 @@ export class GlobalAutoModeService { ); } } + + /** + * Reconcile all feature states for a project on server startup. + * + * Resets features stuck in transient states (in_progress, interrupted, pipeline_*) + * back to a resting state and emits events so the UI reflects corrected states. + * + * This should be called during server initialization to handle: + * - Clean shutdown: features already marked as interrupted + * - Forced kill / crash: features left in in_progress or pipeline_* states + * + * @param projectPath - The project path to reconcile + * @returns The number of features that were reconciled + */ + async reconcileFeatureStates(projectPath: string): Promise { + return this.featureStateManager.reconcileAllFeatureStates(projectPath); + } } diff --git a/apps/server/src/services/feature-state-manager.ts b/apps/server/src/services/feature-state-manager.ts index b33f6df6..4ccc5e5c 100644 --- a/apps/server/src/services/feature-state-manager.ts +++ b/apps/server/src/services/feature-state-manager.ts @@ -273,6 +273,8 @@ export class FeatureStateManager { * * Resets: * - in_progress features back to ready (if has plan) or backlog (if no plan) + * - interrupted features back to ready (if has plan) or backlog (if no plan) + * - pipeline_* features back to ready (if has plan) or backlog (if no plan) * - generating planSpec status back to pending * - in_progress tasks back to pending * @@ -300,6 +302,7 @@ export class FeatureStateManager { if (!feature) continue; let needsUpdate = false; + const originalStatus = feature.status; // Reset in_progress features back to ready/backlog if (feature.status === 'in_progress') { @@ -311,6 +314,30 @@ export class FeatureStateManager { ); } + // Reset interrupted features back to ready/backlog + // These were marked interrupted during graceful shutdown but need to be reset + // so they appear in the correct column and can be re-executed + if (feature.status === 'interrupted') { + const hasApprovedPlan = feature.planSpec?.status === 'approved'; + feature.status = hasApprovedPlan ? 'ready' : 'backlog'; + needsUpdate = true; + logger.info( + `[resetStuckFeatures] Reset feature ${feature.id} from interrupted to ${feature.status}` + ); + } + + // Reset pipeline_* features back to ready/backlog + // After a server restart, pipeline execution cannot resume from the exact step, + // so these need to be reset to a clean state for re-execution + if (feature.status && feature.status.startsWith('pipeline_')) { + const hasApprovedPlan = feature.planSpec?.status === 'approved'; + feature.status = hasApprovedPlan ? 'ready' : 'backlog'; + needsUpdate = true; + logger.info( + `[resetStuckFeatures] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}` + ); + } + // Reset generating planSpec status back to pending (spec generation was interrupted) if (feature.planSpec?.status === 'generating') { feature.planSpec.status = 'pending'; @@ -358,6 +385,130 @@ export class FeatureStateManager { } } + /** + * Reconcile all feature states on server startup. + * + * This method resets all features stuck in transient states (in_progress, + * interrupted, pipeline_*) and emits events so connected UI clients + * immediately reflect the corrected states. + * + * Should be called once during server initialization, before the UI is served, + * to ensure feature state consistency after any type of restart (clean, forced, crash). + * + * @param projectPath - The project path to reconcile features for + * @returns The number of features that were reconciled + */ + async reconcileAllFeatureStates(projectPath: string): Promise { + logger.info(`[reconcileAllFeatureStates] Starting reconciliation for ${projectPath}`); + + const featuresDir = getFeaturesDir(projectPath); + let featuresScanned = 0; + let featuresReconciled = 0; + const reconciledFeatureIds: string[] = []; + + try { + const entries = await secureFs.readdir(featuresDir, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + + featuresScanned++; + const featurePath = path.join(featuresDir, entry.name, 'feature.json'); + const result = await readJsonWithRecovery(featurePath, null, { + maxBackups: DEFAULT_BACKUP_COUNT, + autoRestore: true, + }); + + const feature = result.data; + if (!feature) continue; + + let needsUpdate = false; + const originalStatus = feature.status; + + // Reset features in active execution states back to a resting state + // After a server restart, no processes are actually running + const isActiveState = + feature.status === 'in_progress' || + feature.status === 'interrupted' || + (feature.status && feature.status.startsWith('pipeline_')); + + if (isActiveState) { + const hasApprovedPlan = feature.planSpec?.status === 'approved'; + feature.status = hasApprovedPlan ? 'ready' : 'backlog'; + needsUpdate = true; + logger.info( + `[reconcileAllFeatureStates] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}` + ); + } + + // Reset generating planSpec status back to pending + if (feature.planSpec?.status === 'generating') { + feature.planSpec.status = 'pending'; + needsUpdate = true; + logger.info( + `[reconcileAllFeatureStates] Reset feature ${feature.id} planSpec from generating to pending` + ); + } + + // Reset any in_progress tasks back to pending + if (feature.planSpec?.tasks) { + for (const task of feature.planSpec.tasks) { + if (task.status === 'in_progress') { + task.status = 'pending'; + needsUpdate = true; + logger.info( + `[reconcileAllFeatureStates] Reset task ${task.id} for feature ${feature.id} from in_progress to pending` + ); + if (feature.planSpec?.currentTaskId === task.id) { + feature.planSpec.currentTaskId = undefined; + } + } + } + } + + if (needsUpdate) { + feature.updatedAt = new Date().toISOString(); + await atomicWriteJson(featurePath, feature, { backupCount: DEFAULT_BACKUP_COUNT }); + featuresReconciled++; + reconciledFeatureIds.push(feature.id); + + // Emit per-feature status change event so UI invalidates its cache + this.emitAutoModeEvent('feature_status_changed', { + featureId: feature.id, + projectPath, + status: feature.status, + previousStatus: originalStatus, + reason: 'server_restart_reconciliation', + }); + } + } + + // Emit a bulk reconciliation event for the UI + if (featuresReconciled > 0) { + this.emitAutoModeEvent('features_reconciled', { + projectPath, + reconciledCount: featuresReconciled, + reconciledFeatureIds, + message: `Reconciled ${featuresReconciled} feature(s) after server restart`, + }); + } + + logger.info( + `[reconcileAllFeatureStates] Scanned ${featuresScanned} features, reconciled ${featuresReconciled} for ${projectPath}` + ); + + return featuresReconciled; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + logger.error( + `[reconcileAllFeatureStates] Error reconciling features for ${projectPath}:`, + error + ); + } + return 0; + } + } + /** * Update the planSpec of a feature with partial updates. * diff --git a/apps/server/src/services/typed-event-bus.ts b/apps/server/src/services/typed-event-bus.ts index 11424826..42600db5 100644 --- a/apps/server/src/services/typed-event-bus.ts +++ b/apps/server/src/services/typed-event-bus.ts @@ -42,6 +42,8 @@ export type AutoModeEventType = | 'plan_revision_warning' | 'pipeline_step_started' | 'pipeline_step_complete' + | 'feature_status_changed' + | 'features_reconciled' | string; // Allow other strings for extensibility /** diff --git a/apps/ui/src/hooks/use-query-invalidation.ts b/apps/ui/src/hooks/use-query-invalidation.ts index 0c09977c..241538e3 100644 --- a/apps/ui/src/hooks/use-query-invalidation.ts +++ b/apps/ui/src/hooks/use-query-invalidation.ts @@ -38,6 +38,8 @@ const FEATURE_LIST_INVALIDATION_EVENTS: AutoModeEvent['type'][] = [ 'plan_rejected', 'pipeline_step_started', 'pipeline_step_complete', + 'feature_status_changed', + 'features_reconciled', ]; /** diff --git a/apps/ui/src/types/electron.d.ts b/apps/ui/src/types/electron.d.ts index cf41dabe..e44f19dd 100644 --- a/apps/ui/src/types/electron.d.ts +++ b/apps/ui/src/types/electron.d.ts @@ -359,6 +359,21 @@ export type AutoModeEvent = title?: string; status?: string; }>; + } + | { + type: 'feature_status_changed'; + featureId: string; + projectPath?: string; + status: string; + previousStatus: string; + reason?: string; + } + | { + type: 'features_reconciled'; + projectPath?: string; + reconciledCount: number; + reconciledFeatureIds: string[]; + message: string; }; export type SpecRegenerationEvent =