mirror of
https://github.com/AutoMaker-Org/automaker.git
synced 2026-03-16 21:53:07 +00:00
feat: Add feature state reconciliation on server startup
This commit is contained in:
@@ -386,6 +386,30 @@ eventHookService.initialize(events, settingsService, eventHistoryService, featur
|
||||
await agentService.initialize();
|
||||
logger.info('Agent service initialized');
|
||||
|
||||
// Reconcile feature states on startup
|
||||
// After any type of restart (clean, forced, crash), features may be stuck in
|
||||
// transient states (in_progress, interrupted, pipeline_*) that don't match reality.
|
||||
// Reconcile them back to resting states before the UI is served.
|
||||
try {
|
||||
const settings = await settingsService.getGlobalSettings();
|
||||
if (settings.projects && settings.projects.length > 0) {
|
||||
let totalReconciled = 0;
|
||||
for (const project of settings.projects) {
|
||||
const count = await autoModeService.reconcileFeatureStates(project.path);
|
||||
totalReconciled += count;
|
||||
}
|
||||
if (totalReconciled > 0) {
|
||||
logger.info(
|
||||
`[STARTUP] Reconciled ${totalReconciled} feature(s) across ${settings.projects.length} project(s)`
|
||||
);
|
||||
} else {
|
||||
logger.info('[STARTUP] Feature state reconciliation complete - no stale states found');
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('[STARTUP] Failed to reconcile feature states:', err);
|
||||
}
|
||||
|
||||
// Bootstrap Codex model cache in background (don't block server startup)
|
||||
void codexModelCacheService.getModels().catch((err) => {
|
||||
logger.error('Failed to bootstrap Codex model cache:', err);
|
||||
|
||||
@@ -21,6 +21,7 @@ import { createFollowUpFeatureHandler } from './routes/follow-up-feature.js';
|
||||
import { createCommitFeatureHandler } from './routes/commit-feature.js';
|
||||
import { createApprovePlanHandler } from './routes/approve-plan.js';
|
||||
import { createResumeInterruptedHandler } from './routes/resume-interrupted.js';
|
||||
import { createReconcileHandler } from './routes/reconcile.js';
|
||||
|
||||
/**
|
||||
* Create auto-mode routes.
|
||||
@@ -81,6 +82,11 @@ export function createAutoModeRoutes(autoModeService: AutoModeServiceCompat): Ro
|
||||
validatePathParams('projectPath'),
|
||||
createResumeInterruptedHandler(autoModeService)
|
||||
);
|
||||
router.post(
|
||||
'/reconcile',
|
||||
validatePathParams('projectPath'),
|
||||
createReconcileHandler(autoModeService)
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
53
apps/server/src/routes/auto-mode/routes/reconcile.ts
Normal file
53
apps/server/src/routes/auto-mode/routes/reconcile.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Reconcile Feature States Handler
|
||||
*
|
||||
* On-demand endpoint to reconcile all feature states for a project.
|
||||
* Resets features stuck in transient states (in_progress, interrupted, pipeline_*)
|
||||
* back to resting states (ready/backlog) and emits events to update the UI.
|
||||
*
|
||||
* This is useful when:
|
||||
* - The UI reconnects after a server restart
|
||||
* - A client detects stale feature states
|
||||
* - An admin wants to force-reset stuck features
|
||||
*/
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import { createLogger } from '@automaker/utils';
|
||||
import type { AutoModeServiceCompat } from '../../../services/auto-mode/index.js';
|
||||
|
||||
const logger = createLogger('ReconcileFeatures');
|
||||
|
||||
interface ReconcileRequest {
|
||||
projectPath: string;
|
||||
}
|
||||
|
||||
export function createReconcileHandler(autoModeService: AutoModeServiceCompat) {
|
||||
return async (req: Request, res: Response): Promise<void> => {
|
||||
const { projectPath } = req.body as ReconcileRequest;
|
||||
|
||||
if (!projectPath) {
|
||||
res.status(400).json({ error: 'Project path is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Reconciling feature states for ${projectPath}`);
|
||||
|
||||
try {
|
||||
const reconciledCount = await autoModeService.reconcileFeatureStates(projectPath);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
reconciledCount,
|
||||
message:
|
||||
reconciledCount > 0
|
||||
? `Reconciled ${reconciledCount} feature(s)`
|
||||
: 'No features needed reconciliation',
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Error reconciling feature states:', error);
|
||||
res.status(500).json({
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -88,6 +88,10 @@ export class AutoModeServiceCompat {
|
||||
return this.globalService.markAllRunningFeaturesInterrupted(reason);
|
||||
}
|
||||
|
||||
async reconcileFeatureStates(projectPath: string): Promise<number> {
|
||||
return this.globalService.reconcileFeatureStates(projectPath);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// PER-PROJECT OPERATIONS (delegated to facades)
|
||||
// ===========================================================================
|
||||
|
||||
@@ -205,4 +205,21 @@ export class GlobalAutoModeService {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile all feature states for a project on server startup.
|
||||
*
|
||||
* Resets features stuck in transient states (in_progress, interrupted, pipeline_*)
|
||||
* back to a resting state and emits events so the UI reflects corrected states.
|
||||
*
|
||||
* This should be called during server initialization to handle:
|
||||
* - Clean shutdown: features already marked as interrupted
|
||||
* - Forced kill / crash: features left in in_progress or pipeline_* states
|
||||
*
|
||||
* @param projectPath - The project path to reconcile
|
||||
* @returns The number of features that were reconciled
|
||||
*/
|
||||
async reconcileFeatureStates(projectPath: string): Promise<number> {
|
||||
return this.featureStateManager.reconcileAllFeatureStates(projectPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,6 +273,8 @@ export class FeatureStateManager {
|
||||
*
|
||||
* Resets:
|
||||
* - in_progress features back to ready (if has plan) or backlog (if no plan)
|
||||
* - interrupted features back to ready (if has plan) or backlog (if no plan)
|
||||
* - pipeline_* features back to ready (if has plan) or backlog (if no plan)
|
||||
* - generating planSpec status back to pending
|
||||
* - in_progress tasks back to pending
|
||||
*
|
||||
@@ -300,6 +302,7 @@ export class FeatureStateManager {
|
||||
if (!feature) continue;
|
||||
|
||||
let needsUpdate = false;
|
||||
const originalStatus = feature.status;
|
||||
|
||||
// Reset in_progress features back to ready/backlog
|
||||
if (feature.status === 'in_progress') {
|
||||
@@ -311,6 +314,30 @@ export class FeatureStateManager {
|
||||
);
|
||||
}
|
||||
|
||||
// Reset interrupted features back to ready/backlog
|
||||
// These were marked interrupted during graceful shutdown but need to be reset
|
||||
// so they appear in the correct column and can be re-executed
|
||||
if (feature.status === 'interrupted') {
|
||||
const hasApprovedPlan = feature.planSpec?.status === 'approved';
|
||||
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
|
||||
needsUpdate = true;
|
||||
logger.info(
|
||||
`[resetStuckFeatures] Reset feature ${feature.id} from interrupted to ${feature.status}`
|
||||
);
|
||||
}
|
||||
|
||||
// Reset pipeline_* features back to ready/backlog
|
||||
// After a server restart, pipeline execution cannot resume from the exact step,
|
||||
// so these need to be reset to a clean state for re-execution
|
||||
if (feature.status && feature.status.startsWith('pipeline_')) {
|
||||
const hasApprovedPlan = feature.planSpec?.status === 'approved';
|
||||
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
|
||||
needsUpdate = true;
|
||||
logger.info(
|
||||
`[resetStuckFeatures] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}`
|
||||
);
|
||||
}
|
||||
|
||||
// Reset generating planSpec status back to pending (spec generation was interrupted)
|
||||
if (feature.planSpec?.status === 'generating') {
|
||||
feature.planSpec.status = 'pending';
|
||||
@@ -358,6 +385,130 @@ export class FeatureStateManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile all feature states on server startup.
|
||||
*
|
||||
* This method resets all features stuck in transient states (in_progress,
|
||||
* interrupted, pipeline_*) and emits events so connected UI clients
|
||||
* immediately reflect the corrected states.
|
||||
*
|
||||
* Should be called once during server initialization, before the UI is served,
|
||||
* to ensure feature state consistency after any type of restart (clean, forced, crash).
|
||||
*
|
||||
* @param projectPath - The project path to reconcile features for
|
||||
* @returns The number of features that were reconciled
|
||||
*/
|
||||
async reconcileAllFeatureStates(projectPath: string): Promise<number> {
|
||||
logger.info(`[reconcileAllFeatureStates] Starting reconciliation for ${projectPath}`);
|
||||
|
||||
const featuresDir = getFeaturesDir(projectPath);
|
||||
let featuresScanned = 0;
|
||||
let featuresReconciled = 0;
|
||||
const reconciledFeatureIds: string[] = [];
|
||||
|
||||
try {
|
||||
const entries = await secureFs.readdir(featuresDir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
|
||||
featuresScanned++;
|
||||
const featurePath = path.join(featuresDir, entry.name, 'feature.json');
|
||||
const result = await readJsonWithRecovery<Feature | null>(featurePath, null, {
|
||||
maxBackups: DEFAULT_BACKUP_COUNT,
|
||||
autoRestore: true,
|
||||
});
|
||||
|
||||
const feature = result.data;
|
||||
if (!feature) continue;
|
||||
|
||||
let needsUpdate = false;
|
||||
const originalStatus = feature.status;
|
||||
|
||||
// Reset features in active execution states back to a resting state
|
||||
// After a server restart, no processes are actually running
|
||||
const isActiveState =
|
||||
feature.status === 'in_progress' ||
|
||||
feature.status === 'interrupted' ||
|
||||
(feature.status && feature.status.startsWith('pipeline_'));
|
||||
|
||||
if (isActiveState) {
|
||||
const hasApprovedPlan = feature.planSpec?.status === 'approved';
|
||||
feature.status = hasApprovedPlan ? 'ready' : 'backlog';
|
||||
needsUpdate = true;
|
||||
logger.info(
|
||||
`[reconcileAllFeatureStates] Reset feature ${feature.id} from ${originalStatus} to ${feature.status}`
|
||||
);
|
||||
}
|
||||
|
||||
// Reset generating planSpec status back to pending
|
||||
if (feature.planSpec?.status === 'generating') {
|
||||
feature.planSpec.status = 'pending';
|
||||
needsUpdate = true;
|
||||
logger.info(
|
||||
`[reconcileAllFeatureStates] Reset feature ${feature.id} planSpec from generating to pending`
|
||||
);
|
||||
}
|
||||
|
||||
// Reset any in_progress tasks back to pending
|
||||
if (feature.planSpec?.tasks) {
|
||||
for (const task of feature.planSpec.tasks) {
|
||||
if (task.status === 'in_progress') {
|
||||
task.status = 'pending';
|
||||
needsUpdate = true;
|
||||
logger.info(
|
||||
`[reconcileAllFeatureStates] Reset task ${task.id} for feature ${feature.id} from in_progress to pending`
|
||||
);
|
||||
if (feature.planSpec?.currentTaskId === task.id) {
|
||||
feature.planSpec.currentTaskId = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (needsUpdate) {
|
||||
feature.updatedAt = new Date().toISOString();
|
||||
await atomicWriteJson(featurePath, feature, { backupCount: DEFAULT_BACKUP_COUNT });
|
||||
featuresReconciled++;
|
||||
reconciledFeatureIds.push(feature.id);
|
||||
|
||||
// Emit per-feature status change event so UI invalidates its cache
|
||||
this.emitAutoModeEvent('feature_status_changed', {
|
||||
featureId: feature.id,
|
||||
projectPath,
|
||||
status: feature.status,
|
||||
previousStatus: originalStatus,
|
||||
reason: 'server_restart_reconciliation',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Emit a bulk reconciliation event for the UI
|
||||
if (featuresReconciled > 0) {
|
||||
this.emitAutoModeEvent('features_reconciled', {
|
||||
projectPath,
|
||||
reconciledCount: featuresReconciled,
|
||||
reconciledFeatureIds,
|
||||
message: `Reconciled ${featuresReconciled} feature(s) after server restart`,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[reconcileAllFeatureStates] Scanned ${featuresScanned} features, reconciled ${featuresReconciled} for ${projectPath}`
|
||||
);
|
||||
|
||||
return featuresReconciled;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
logger.error(
|
||||
`[reconcileAllFeatureStates] Error reconciling features for ${projectPath}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the planSpec of a feature with partial updates.
|
||||
*
|
||||
|
||||
@@ -42,6 +42,8 @@ export type AutoModeEventType =
|
||||
| 'plan_revision_warning'
|
||||
| 'pipeline_step_started'
|
||||
| 'pipeline_step_complete'
|
||||
| 'feature_status_changed'
|
||||
| 'features_reconciled'
|
||||
| string; // Allow other strings for extensibility
|
||||
|
||||
/**
|
||||
|
||||
@@ -38,6 +38,8 @@ const FEATURE_LIST_INVALIDATION_EVENTS: AutoModeEvent['type'][] = [
|
||||
'plan_rejected',
|
||||
'pipeline_step_started',
|
||||
'pipeline_step_complete',
|
||||
'feature_status_changed',
|
||||
'features_reconciled',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
15
apps/ui/src/types/electron.d.ts
vendored
15
apps/ui/src/types/electron.d.ts
vendored
@@ -359,6 +359,21 @@ export type AutoModeEvent =
|
||||
title?: string;
|
||||
status?: string;
|
||||
}>;
|
||||
}
|
||||
| {
|
||||
type: 'feature_status_changed';
|
||||
featureId: string;
|
||||
projectPath?: string;
|
||||
status: string;
|
||||
previousStatus: string;
|
||||
reason?: string;
|
||||
}
|
||||
| {
|
||||
type: 'features_reconciled';
|
||||
projectPath?: string;
|
||||
reconciledCount: number;
|
||||
reconciledFeatureIds: string[];
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type SpecRegenerationEvent =
|
||||
|
||||
Reference in New Issue
Block a user