feat: Add feature state reconciliation on server startup

This commit is contained in:
gsxdsm
2026-02-17 09:56:54 -08:00
parent 57446b4fba
commit b5ad77b0f9
9 changed files with 274 additions and 0 deletions

View File

@@ -386,6 +386,30 @@ eventHookService.initialize(events, settingsService, eventHistoryService, featur
await agentService.initialize();
logger.info('Agent service initialized');
// Reconcile feature states on startup
// After any type of restart (clean, forced, crash), features may be stuck in
// transient states (in_progress, interrupted, pipeline_*) that don't match reality.
// Reconcile them back to resting states before the UI is served.
try {
const settings = await settingsService.getGlobalSettings();
if (settings.projects && settings.projects.length > 0) {
let totalReconciled = 0;
for (const project of settings.projects) {
const count = await autoModeService.reconcileFeatureStates(project.path);
totalReconciled += count;
}
if (totalReconciled > 0) {
logger.info(
`[STARTUP] Reconciled ${totalReconciled} feature(s) across ${settings.projects.length} project(s)`
);
} else {
logger.info('[STARTUP] Feature state reconciliation complete - no stale states found');
}
}
} catch (err) {
logger.warn('[STARTUP] Failed to reconcile feature states:', err);
}
// Bootstrap Codex model cache in background (don't block server startup)
void codexModelCacheService.getModels().catch((err) => {
logger.error('Failed to bootstrap Codex model cache:', err);

View File

@@ -21,6 +21,7 @@ import { createFollowUpFeatureHandler } from './routes/follow-up-feature.js';
import { createCommitFeatureHandler } from './routes/commit-feature.js';
import { createApprovePlanHandler } from './routes/approve-plan.js';
import { createResumeInterruptedHandler } from './routes/resume-interrupted.js';
import { createReconcileHandler } from './routes/reconcile.js';
/**
* Create auto-mode routes.
@@ -81,6 +82,11 @@ export function createAutoModeRoutes(autoModeService: AutoModeServiceCompat): Ro
validatePathParams('projectPath'),
createResumeInterruptedHandler(autoModeService)
);
router.post(
'/reconcile',
validatePathParams('projectPath'),
createReconcileHandler(autoModeService)
);
return router;
}

View File

@@ -0,0 +1,53 @@
/**
* Reconcile Feature States Handler
*
* On-demand endpoint to reconcile all feature states for a project.
* Resets features stuck in transient states (in_progress, interrupted, pipeline_*)
* back to resting states (ready/backlog) and emits events to update the UI.
*
* This is useful when:
* - The UI reconnects after a server restart
* - A client detects stale feature states
* - An admin wants to force-reset stuck features
*/
import type { Request, Response } from 'express';
import { createLogger } from '@automaker/utils';
import type { AutoModeServiceCompat } from '../../../services/auto-mode/index.js';
const logger = createLogger('ReconcileFeatures');
interface ReconcileRequest {
projectPath: string;
}
export function createReconcileHandler(autoModeService: AutoModeServiceCompat) {
return async (req: Request, res: Response): Promise<void> => {
const { projectPath } = req.body as ReconcileRequest;
if (!projectPath) {
res.status(400).json({ error: 'Project path is required' });
return;
}
logger.info(`Reconciling feature states for ${projectPath}`);
try {
const reconciledCount = await autoModeService.reconcileFeatureStates(projectPath);
res.json({
success: true,
reconciledCount,
message:
reconciledCount > 0
? `Reconciled ${reconciledCount} feature(s)`
: 'No features needed reconciliation',
});
} catch (error) {
logger.error('Error reconciling feature states:', error);
res.status(500).json({
error: error instanceof Error ? error.message : 'Unknown error',
});
}
};
}

View File

@@ -88,6 +88,10 @@ export class AutoModeServiceCompat {
return this.globalService.markAllRunningFeaturesInterrupted(reason);
}
async reconcileFeatureStates(projectPath: string): Promise<number> {
return this.globalService.reconcileFeatureStates(projectPath);
}
// ===========================================================================
// PER-PROJECT OPERATIONS (delegated to facades)
// ===========================================================================

View File

@@ -205,4 +205,21 @@ export class GlobalAutoModeService {
);
}
}
/**
* Reconcile all feature states for a project on server startup.
*
* Resets features stuck in transient states (in_progress, interrupted, pipeline_*)
* back to a resting state and emits events so the UI reflects corrected states.
*
* This should be called during server initialization to handle:
* - Clean shutdown: features already marked as interrupted
* - Forced kill / crash: features left in in_progress or pipeline_* states
*
* @param projectPath - The project path to reconcile
* @returns The number of features that were reconciled
*/
async reconcileFeatureStates(projectPath: string): Promise<number> {
return this.featureStateManager.reconcileAllFeatureStates(projectPath);
}
}

View File

@@ -273,6 +273,8 @@ export class FeatureStateManager {
*
* Resets:
* - in_progress features back to ready (if has plan) or backlog (if no plan)
* - interrupted features back to ready (if has plan) or backlog (if no plan)
* - pipeline_* features back to ready (if has plan) or backlog (if no plan)
* - generating planSpec status back to pending
* - in_progress tasks back to pending
*
@@ -300,6 +302,7 @@ export class FeatureStateManager {
if (!feature) continue;
let needsUpdate = false;
const originalStatus = feature.status;
// Reset in_progress features back to ready/backlog
if (feature.status === 'in_progress') {
@@ -311,6 +314,30 @@ export class FeatureStateManager {
);
}
// Reset interrupted features back to ready/backlog
// These were marked interrupted during graceful shutdown but need to be reset
// so they appear in the correct column and can be re-executed
if (feature.status === 'interrupted') {
const hasApprovedPlan = feature.planSpec?.status === 'approved';
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
needsUpdate = true;
logger.info(
`[resetStuckFeatures] Reset feature ${feature.id} from interrupted to ${feature.status}`
);
}
// Reset pipeline_* features back to ready/backlog
// After a server restart, pipeline execution cannot resume from the exact step,
// so these need to be reset to a clean state for re-execution
if (feature.status && feature.status.startsWith('pipeline_')) {
const hasApprovedPlan = feature.planSpec?.status === 'approved';
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
needsUpdate = true;
logger.info(
`[resetStuckFeatures] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}`
);
}
// Reset generating planSpec status back to pending (spec generation was interrupted)
if (feature.planSpec?.status === 'generating') {
feature.planSpec.status = 'pending';
@@ -358,6 +385,130 @@ export class FeatureStateManager {
}
}
/**
* Reconcile all feature states on server startup.
*
* This method resets all features stuck in transient states (in_progress,
* interrupted, pipeline_*) and emits events so connected UI clients
* immediately reflect the corrected states.
*
* Should be called once during server initialization, before the UI is served,
* to ensure feature state consistency after any type of restart (clean, forced, crash).
*
* @param projectPath - The project path to reconcile features for
* @returns The number of features that were reconciled
*/
async reconcileAllFeatureStates(projectPath: string): Promise<number> {
logger.info(`[reconcileAllFeatureStates] Starting reconciliation for ${projectPath}`);
const featuresDir = getFeaturesDir(projectPath);
let featuresScanned = 0;
let featuresReconciled = 0;
const reconciledFeatureIds: string[] = [];
try {
const entries = await secureFs.readdir(featuresDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
featuresScanned++;
const featurePath = path.join(featuresDir, entry.name, 'feature.json');
const result = await readJsonWithRecovery<Feature | null>(featurePath, null, {
maxBackups: DEFAULT_BACKUP_COUNT,
autoRestore: true,
});
const feature = result.data;
if (!feature) continue;
let needsUpdate = false;
const originalStatus = feature.status;
// Reset features in active execution states back to a resting state
// After a server restart, no processes are actually running
const isActiveState =
feature.status === 'in_progress' ||
feature.status === 'interrupted' ||
(feature.status && feature.status.startsWith('pipeline_'));
if (isActiveState) {
const hasApprovedPlan = feature.planSpec?.status === 'approved';
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
needsUpdate = true;
logger.info(
`[reconcileAllFeatureStates] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}`
);
}
// Reset generating planSpec status back to pending
if (feature.planSpec?.status === 'generating') {
feature.planSpec.status = 'pending';
needsUpdate = true;
logger.info(
`[reconcileAllFeatureStates] Reset feature ${feature.id} planSpec from generating to pending`
);
}
// Reset any in_progress tasks back to pending
if (feature.planSpec?.tasks) {
for (const task of feature.planSpec.tasks) {
if (task.status === 'in_progress') {
task.status = 'pending';
needsUpdate = true;
logger.info(
`[reconcileAllFeatureStates] Reset task ${task.id} for feature ${feature.id} from in_progress to pending`
);
if (feature.planSpec?.currentTaskId === task.id) {
feature.planSpec.currentTaskId = undefined;
}
}
}
}
if (needsUpdate) {
feature.updatedAt = new Date().toISOString();
await atomicWriteJson(featurePath, feature, { backupCount: DEFAULT_BACKUP_COUNT });
featuresReconciled++;
reconciledFeatureIds.push(feature.id);
// Emit per-feature status change event so UI invalidates its cache
this.emitAutoModeEvent('feature_status_changed', {
featureId: feature.id,
projectPath,
status: feature.status,
previousStatus: originalStatus,
reason: 'server_restart_reconciliation',
});
}
}
// Emit a bulk reconciliation event for the UI
if (featuresReconciled > 0) {
this.emitAutoModeEvent('features_reconciled', {
projectPath,
reconciledCount: featuresReconciled,
reconciledFeatureIds,
message: `Reconciled ${featuresReconciled} feature(s) after server restart`,
});
}
logger.info(
`[reconcileAllFeatureStates] Scanned ${featuresScanned} features, reconciled ${featuresReconciled} for ${projectPath}`
);
return featuresReconciled;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
logger.error(
`[reconcileAllFeatureStates] Error reconciling features for ${projectPath}:`,
error
);
}
return 0;
}
}
/**
* Update the planSpec of a feature with partial updates.
*

View File

@@ -42,6 +42,8 @@ export type AutoModeEventType =
| 'plan_revision_warning'
| 'pipeline_step_started'
| 'pipeline_step_complete'
| 'feature_status_changed'
| 'features_reconciled'
| string; // Allow other strings for extensibility
/**

View File

@@ -38,6 +38,8 @@ const FEATURE_LIST_INVALIDATION_EVENTS: AutoModeEvent['type'][] = [
'plan_rejected',
'pipeline_step_started',
'pipeline_step_complete',
'feature_status_changed',
'features_reconciled',
];
/**

View File

@@ -359,6 +359,21 @@ export type AutoModeEvent =
title?: string;
status?: string;
}>;
}
| {
type: 'feature_status_changed';
featureId: string;
projectPath?: string;
status: string;
previousStatus: string;
reason?: string;
}
| {
type: 'features_reconciled';
projectPath?: string;
reconciledCount: number;
reconciledFeatureIds: string[];
message: string;
};
export type SpecRegenerationEvent =