From 7cbdb3db7344dadc049e260915169737ba6c7fce Mon Sep 17 00:00:00 2001 From: Kacper Date: Sat, 13 Dec 2025 04:26:58 +0100 Subject: [PATCH] refactor: eliminate code duplication with shared utilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created 5 new utility modules in apps/server/src/lib/ to eliminate ~320 lines of duplicated code: - image-handler.ts: Centralized image processing (MIME types, base64, content blocks) - prompt-builder.ts: Standardized prompt building with image attachments - model-resolver.ts: Model alias resolution and provider routing - conversation-utils.ts: Conversation history processing for providers - error-handler.ts: Error classification and user-friendly messages Updated services and providers to use shared utilities: - agent-service.ts: -51 lines (removed duplicate image handling, model logic) - auto-mode-service.ts: -75 lines (removed MODEL_MAP, duplicate utilities) - claude-provider.ts: -10 lines (uses conversation-utils) - codex-provider.ts: -5 lines (uses conversation-utils) Added comprehensive documentation: - docs/server/utilities.md: Complete reference for all 9 lib utilities - docs/server/providers.md: Provider architecture guide with examples Benefits: - Single source of truth for critical business logic - Improved maintainability and testability - Consistent behavior across services and providers - Better documentation for future development 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- apps/server/src/lib/conversation-utils.ts | 97 +++ apps/server/src/lib/error-handler.ts | 104 +++ apps/server/src/lib/image-handler.ts | 135 +++ apps/server/src/lib/model-resolver.ts | 88 ++ apps/server/src/lib/prompt-builder.ts | 79 ++ apps/server/src/providers/claude-provider.ts | 22 +- apps/server/src/providers/codex-provider.ts | 11 +- apps/server/src/services/agent-service.ts | 89 +- apps/server/src/services/auto-mode-service.ts | 113 +-- docs/server/providers.md | 786 ++++++++++++++++++ docs/server/utilities.md | 672 +++++++++++++++ 11 files changed, 2008 insertions(+), 188 deletions(-) create mode 100644 apps/server/src/lib/conversation-utils.ts create mode 100644 apps/server/src/lib/error-handler.ts create mode 100644 apps/server/src/lib/image-handler.ts create mode 100644 apps/server/src/lib/model-resolver.ts create mode 100644 apps/server/src/lib/prompt-builder.ts create mode 100644 docs/server/providers.md create mode 100644 docs/server/utilities.md diff --git a/apps/server/src/lib/conversation-utils.ts b/apps/server/src/lib/conversation-utils.ts new file mode 100644 index 00000000..3fe95a60 --- /dev/null +++ b/apps/server/src/lib/conversation-utils.ts @@ -0,0 +1,97 @@ +/** + * Conversation history utilities for processing message history + * + * Provides standardized conversation history handling: + * - Extract text from content (string or array format) + * - Normalize content blocks to array format + * - Format history as plain text for CLI-based providers + * - Convert history to Claude SDK message format + */ + +import type { ConversationMessage } from "../providers/types.js"; + +/** + * Extract plain text from message content (handles both string and array formats) + * + * @param content - Message content (string or array of content blocks) + * @returns Extracted text content + */ +export function extractTextFromContent( + content: string | Array<{ type: string; text?: string; source?: object }> +): string { + if (typeof content === "string") { + return content; + } + + // Extract text blocks only + return content + .filter((block) => block.type === "text") + .map((block) => block.text || "") + .join("\n"); +} + +/** + * Normalize message content to array format + * + * @param content - Message content (string or array) + * @returns Content as array of blocks + */ +export function normalizeContentBlocks( + content: string | Array<{ type: string; text?: string; source?: object }> +): Array<{ type: string; text?: string; source?: object }> { + if (Array.isArray(content)) { + return content; + } + return [{ type: "text", text: content }]; +} + +/** + * Format conversation history as plain text for CLI-based providers + * + * @param history - Array of conversation messages + * @returns Formatted text with role labels + */ +export function formatHistoryAsText(history: ConversationMessage[]): string { + if (history.length === 0) { + return ""; + } + + let historyText = "Previous conversation:\n\n"; + + for (const msg of history) { + const contentText = extractTextFromContent(msg.content); + const role = msg.role === "user" ? "User" : "Assistant"; + historyText += `${role}: ${contentText}\n\n`; + } + + historyText += "---\n\n"; + return historyText; +} + +/** + * Convert conversation history to Claude SDK message format + * + * @param history - Array of conversation messages + * @returns Array of Claude SDK formatted messages + */ +export function convertHistoryToMessages( + history: ConversationMessage[] +): Array<{ + type: "user" | "assistant"; + session_id: string; + message: { + role: "user" | "assistant"; + content: Array<{ type: string; text?: string; source?: object }>; + }; + parent_tool_use_id: null; +}> { + return history.map((historyMsg) => ({ + type: historyMsg.role, + session_id: "", + message: { + role: historyMsg.role, + content: normalizeContentBlocks(historyMsg.content), + }, + parent_tool_use_id: null, + })); +} diff --git a/apps/server/src/lib/error-handler.ts b/apps/server/src/lib/error-handler.ts new file mode 100644 index 00000000..b1e0c3ac --- /dev/null +++ b/apps/server/src/lib/error-handler.ts @@ -0,0 +1,104 @@ +/** + * Error handling utilities for standardized error classification + * + * Provides utilities for: + * - Detecting abort/cancellation errors + * - Detecting authentication errors + * - Classifying errors by type + * - Generating user-friendly error messages + */ + +/** + * Check if an error is an abort/cancellation error + * + * @param error - The error to check + * @returns True if the error is an abort error + */ +export function isAbortError(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === "AbortError" || error.message.includes("abort")) + ); +} + +/** + * Check if an error is an authentication/API key error + * + * @param errorMessage - The error message to check + * @returns True if the error is authentication-related + */ +export function isAuthenticationError(errorMessage: string): boolean { + return ( + errorMessage.includes("Authentication failed") || + errorMessage.includes("Invalid API key") || + errorMessage.includes("authentication_failed") || + errorMessage.includes("Fix external API key") + ); +} + +/** + * Error type classification + */ +export type ErrorType = "authentication" | "abort" | "execution" | "unknown"; + +/** + * Classified error information + */ +export interface ErrorInfo { + type: ErrorType; + message: string; + isAbort: boolean; + isAuth: boolean; + originalError: unknown; +} + +/** + * Classify an error into a specific type + * + * @param error - The error to classify + * @returns Classified error information + */ +export function classifyError(error: unknown): ErrorInfo { + const message = error instanceof Error ? error.message : String(error || "Unknown error"); + const isAbort = isAbortError(error); + const isAuth = isAuthenticationError(message); + + let type: ErrorType; + if (isAuth) { + type = "authentication"; + } else if (isAbort) { + type = "abort"; + } else if (error instanceof Error) { + type = "execution"; + } else { + type = "unknown"; + } + + return { + type, + message, + isAbort, + isAuth, + originalError: error, + }; +} + +/** + * Get a user-friendly error message + * + * @param error - The error to convert + * @returns User-friendly error message + */ +export function getUserFriendlyErrorMessage(error: unknown): string { + const info = classifyError(error); + + if (info.isAbort) { + return "Operation was cancelled"; + } + + if (info.isAuth) { + return "Authentication failed. Please check your API key."; + } + + return info.message; +} diff --git a/apps/server/src/lib/image-handler.ts b/apps/server/src/lib/image-handler.ts new file mode 100644 index 00000000..167f948f --- /dev/null +++ b/apps/server/src/lib/image-handler.ts @@ -0,0 +1,135 @@ +/** + * Image handling utilities for processing image files + * + * Provides utilities for: + * - MIME type detection based on file extensions + * - Base64 encoding of image files + * - Content block generation for Claude SDK format + * - Path resolution (relative/absolute) + */ + +import fs from "fs/promises"; +import path from "path"; + +/** + * MIME type mapping for image file extensions + */ +const IMAGE_MIME_TYPES: Record = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", +} as const; + +/** + * Image data with base64 encoding and metadata + */ +export interface ImageData { + base64: string; + mimeType: string; + filename: string; + originalPath: string; +} + +/** + * Content block for image (Claude SDK format) + */ +export interface ImageContentBlock { + type: "image"; + source: { + type: "base64"; + media_type: string; + data: string; + }; +} + +/** + * Get MIME type for an image file based on extension + * + * @param imagePath - Path to the image file + * @returns MIME type string (defaults to "image/png" for unknown extensions) + */ +export function getMimeTypeForImage(imagePath: string): string { + const ext = path.extname(imagePath).toLowerCase(); + return IMAGE_MIME_TYPES[ext] || "image/png"; +} + +/** + * Read an image file and convert to base64 with metadata + * + * @param imagePath - Path to the image file + * @returns Promise resolving to image data with base64 encoding + * @throws Error if file cannot be read + */ +export async function readImageAsBase64(imagePath: string): Promise { + const imageBuffer = await fs.readFile(imagePath); + const base64Data = imageBuffer.toString("base64"); + const mimeType = getMimeTypeForImage(imagePath); + + return { + base64: base64Data, + mimeType, + filename: path.basename(imagePath), + originalPath: imagePath, + }; +} + +/** + * Convert image paths to content blocks (Claude SDK format) + * Handles both relative and absolute paths + * + * @param imagePaths - Array of image file paths + * @param workDir - Optional working directory for resolving relative paths + * @returns Promise resolving to array of image content blocks + */ +export async function convertImagesToContentBlocks( + imagePaths: string[], + workDir?: string +): Promise { + const blocks: ImageContentBlock[] = []; + + for (const imagePath of imagePaths) { + try { + // Resolve to absolute path if needed + const absolutePath = workDir && !path.isAbsolute(imagePath) + ? path.join(workDir, imagePath) + : imagePath; + + const imageData = await readImageAsBase64(absolutePath); + + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: imageData.mimeType, + data: imageData.base64, + }, + }); + } catch (error) { + console.error(`[ImageHandler] Failed to load image ${imagePath}:`, error); + // Continue processing other images + } + } + + return blocks; +} + +/** + * Build a list of image paths for text prompts + * Formats image paths as a bulleted list for inclusion in text prompts + * + * @param imagePaths - Array of image file paths + * @returns Formatted string with image paths, or empty string if no images + */ +export function formatImagePathsForPrompt(imagePaths: string[]): string { + if (imagePaths.length === 0) { + return ""; + } + + let text = "\n\nAttached images:\n"; + for (const imagePath of imagePaths) { + text += `- ${imagePath}\n`; + } + return text; +} diff --git a/apps/server/src/lib/model-resolver.ts b/apps/server/src/lib/model-resolver.ts new file mode 100644 index 00000000..f524dbdf --- /dev/null +++ b/apps/server/src/lib/model-resolver.ts @@ -0,0 +1,88 @@ +/** + * Model resolution utilities for handling model string mapping + * + * Provides centralized model resolution logic: + * - Maps Claude model aliases to full model strings + * - Detects and passes through OpenAI/Codex models + * - Provides default models per provider + * - Handles multiple model sources with priority + */ + +/** + * Model alias mapping for Claude models + */ +export const CLAUDE_MODEL_MAP: Record = { + haiku: "claude-haiku-4-5", + sonnet: "claude-sonnet-4-20250514", + opus: "claude-opus-4-5-20251101", +} as const; + +/** + * Default models per provider + */ +export const DEFAULT_MODELS = { + claude: "claude-opus-4-5-20251101", + openai: "gpt-5.2", +} as const; + +/** + * Resolve a model key/alias to a full model string + * + * @param modelKey - Model key (e.g., "opus", "gpt-5.2", "claude-sonnet-4-20250514") + * @param defaultModel - Fallback model if modelKey is undefined + * @returns Full model string + */ +export function resolveModelString( + modelKey?: string, + defaultModel: string = DEFAULT_MODELS.claude +): string { + // No model specified - use default + if (!modelKey) { + return defaultModel; + } + + // OpenAI/Codex models - pass through unchanged + if (modelKey.startsWith("gpt-") || modelKey.startsWith("o")) { + console.log(`[ModelResolver] Using OpenAI/Codex model: ${modelKey}`); + return modelKey; + } + + // Full Claude model string - pass through unchanged + if (modelKey.includes("claude-")) { + console.log(`[ModelResolver] Using full Claude model string: ${modelKey}`); + return modelKey; + } + + // Look up Claude model alias + const resolved = CLAUDE_MODEL_MAP[modelKey]; + if (resolved) { + console.log(`[ModelResolver] Resolved model alias: "${modelKey}" -> "${resolved}"`); + return resolved; + } + + // Unknown model key - use default + console.warn( + `[ModelResolver] Unknown model key "${modelKey}", using default: "${defaultModel}"` + ); + return defaultModel; +} + +/** + * Get the effective model from multiple sources + * Priority: explicit model > session model > default + * + * @param explicitModel - Explicitly provided model (highest priority) + * @param sessionModel - Model from session (medium priority) + * @param defaultModel - Fallback default model (lowest priority) + * @returns Resolved model string + */ +export function getEffectiveModel( + explicitModel?: string, + sessionModel?: string, + defaultModel?: string +): string { + return resolveModelString( + explicitModel || sessionModel, + defaultModel + ); +} diff --git a/apps/server/src/lib/prompt-builder.ts b/apps/server/src/lib/prompt-builder.ts new file mode 100644 index 00000000..c6ce2e7d --- /dev/null +++ b/apps/server/src/lib/prompt-builder.ts @@ -0,0 +1,79 @@ +/** + * Prompt building utilities for constructing prompts with images + * + * Provides standardized prompt building that: + * - Combines text prompts with image attachments + * - Handles content block array generation + * - Optionally includes image paths in text + * - Supports both vision and non-vision models + */ + +import { convertImagesToContentBlocks, formatImagePathsForPrompt } from "./image-handler.js"; + +/** + * Content that can be either simple text or structured blocks + */ +export type PromptContent = string | Array<{ + type: string; + text?: string; + source?: object; +}>; + +/** + * Result of building a prompt with optional images + */ +export interface PromptWithImages { + content: PromptContent; + hasImages: boolean; +} + +/** + * Build a prompt with optional image attachments + * + * @param basePrompt - The text prompt + * @param imagePaths - Optional array of image file paths + * @param workDir - Optional working directory for resolving relative paths + * @param includeImagePaths - Whether to append image paths to the text (default: false) + * @returns Promise resolving to prompt content and metadata + */ +export async function buildPromptWithImages( + basePrompt: string, + imagePaths?: string[], + workDir?: string, + includeImagePaths: boolean = false +): Promise { + // No images - return plain text + if (!imagePaths || imagePaths.length === 0) { + return { content: basePrompt, hasImages: false }; + } + + // Build text content with optional image path listing + let textContent = basePrompt; + if (includeImagePaths) { + textContent += formatImagePathsForPrompt(imagePaths); + } + + // Build content blocks array + const contentBlocks: Array<{ + type: string; + text?: string; + source?: object; + }> = []; + + // Add text block if we have text + if (textContent.trim()) { + contentBlocks.push({ type: "text", text: textContent }); + } + + // Add image blocks + const imageBlocks = await convertImagesToContentBlocks(imagePaths, workDir); + contentBlocks.push(...imageBlocks); + + // Return appropriate format + const content: PromptContent = + contentBlocks.length > 1 || contentBlocks[0]?.type === "image" + ? contentBlocks + : textContent; + + return { content, hasImages: true }; +} diff --git a/apps/server/src/providers/claude-provider.ts b/apps/server/src/providers/claude-provider.ts index 9384ad34..c1456c63 100644 --- a/apps/server/src/providers/claude-provider.ts +++ b/apps/server/src/providers/claude-provider.ts @@ -7,6 +7,7 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; import { BaseProvider } from "./base-provider.js"; +import { convertHistoryToMessages, normalizeContentBlocks } from "../lib/conversation-utils.js"; import type { ExecuteOptions, ProviderMessage, @@ -64,19 +65,10 @@ export class ClaudeProvider extends BaseProvider { if (conversationHistory && conversationHistory.length > 0) { // Multi-turn conversation with history promptPayload = (async function* () { - // Yield all previous messages first - for (const historyMsg of conversationHistory) { - yield { - type: historyMsg.role, - session_id: "", - message: { - role: historyMsg.role, - content: Array.isArray(historyMsg.content) - ? historyMsg.content - : [{ type: "text", text: historyMsg.content }], - }, - parent_tool_use_id: null, - }; + // Yield history messages using utility + const historyMessages = convertHistoryToMessages(conversationHistory); + for (const msg of historyMessages) { + yield msg; } // Yield current prompt @@ -85,9 +77,7 @@ export class ClaudeProvider extends BaseProvider { session_id: "", message: { role: "user" as const, - content: Array.isArray(prompt) - ? prompt - : [{ type: "text", text: prompt }], + content: normalizeContentBlocks(prompt), }, parent_tool_use_id: null, }; diff --git a/apps/server/src/providers/codex-provider.ts b/apps/server/src/providers/codex-provider.ts index 4739b7c8..531dd268 100644 --- a/apps/server/src/providers/codex-provider.ts +++ b/apps/server/src/providers/codex-provider.ts @@ -9,6 +9,7 @@ import { BaseProvider } from "./base-provider.js"; import { CodexCliDetector } from "./codex-cli-detector.js"; import { codexConfigManager } from "./codex-config-manager.js"; import { spawnJSONLProcess } from "../lib/subprocess-manager.js"; +import { formatHistoryAsText } from "../lib/conversation-utils.js"; import type { ExecuteOptions, ProviderMessage, @@ -99,14 +100,8 @@ export class CodexProvider extends BaseProvider { // Add conversation history if (conversationHistory && conversationHistory.length > 0) { - let historyText = "Previous conversation:\n\n"; - for (const msg of conversationHistory) { - const contentText = typeof msg.content === "string" - ? msg.content - : msg.content.map(c => c.text || "").join(""); - historyText += `${msg.role === "user" ? "User" : "Assistant"}: ${contentText}\n\n`; - } - combinedPrompt = `${historyText}---\n\nCurrent request:\n${combinedPrompt}`; + const historyText = formatHistoryAsText(conversationHistory); + combinedPrompt = `${historyText}Current request:\n${combinedPrompt}`; } // Build command arguments diff --git a/apps/server/src/services/agent-service.ts b/apps/server/src/services/agent-service.ts index b839d452..b1c8cd1b 100644 --- a/apps/server/src/services/agent-service.ts +++ b/apps/server/src/services/agent-service.ts @@ -9,6 +9,12 @@ import fs from "fs/promises"; import type { EventEmitter } from "../lib/events.js"; import { ProviderFactory } from "../providers/provider-factory.js"; import type { ExecuteOptions } from "../providers/types.js"; +import { + readImageAsBase64, +} from "../lib/image-handler.js"; +import { buildPromptWithImages } from "../lib/prompt-builder.js"; +import { getEffectiveModel } from "../lib/model-resolver.js"; +import { isAbortError } from "../lib/error-handler.js"; interface Message { id: string; @@ -123,22 +129,11 @@ export class AgentService { if (imagePaths && imagePaths.length > 0) { for (const imagePath of imagePaths) { try { - const imageBuffer = await fs.readFile(imagePath); - const base64Data = imageBuffer.toString("base64"); - const ext = path.extname(imagePath).toLowerCase(); - const mimeTypeMap: Record = { - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".png": "image/png", - ".gif": "image/gif", - ".webp": "image/webp", - }; - const mediaType = mimeTypeMap[ext] || "image/png"; - + const imageData = await readImageAsBase64(imagePath); images.push({ - data: base64Data, - mimeType: mediaType, - filename: path.basename(imagePath), + data: imageData.base64, + mimeType: imageData.mimeType, + filename: imageData.filename, }); } catch (error) { console.error(`[AgentService] Failed to load image ${imagePath}:`, error); @@ -175,7 +170,7 @@ export class AgentService { try { // Use session model, parameter model, or default - const effectiveModel = model || session.model || "claude-opus-4-5-20251101"; + const effectiveModel = getEffectiveModel(model, session.model); // Get provider for this model const provider = ProviderFactory.getProviderForModel(effectiveModel); @@ -205,59 +200,13 @@ export class AgentService { conversationHistory: conversationHistory.length > 0 ? conversationHistory : undefined, }; - // Build prompt content - let promptContent: string | Array<{ type: string; text?: string; source?: object }> = - message; - - // Append image paths to prompt text (like old implementation) - if (imagePaths && imagePaths.length > 0) { - let enhancedMessage = message; - - // Append image file paths to the message text - enhancedMessage += "\n\nAttached images:\n"; - for (const imagePath of imagePaths) { - enhancedMessage += `- ${imagePath}\n`; - } - - const contentBlocks: Array<{ type: string; text?: string; source?: object }> = []; - - if (enhancedMessage && enhancedMessage.trim()) { - contentBlocks.push({ type: "text", text: enhancedMessage }); - } - - for (const imagePath of imagePaths) { - try { - const imageBuffer = await fs.readFile(imagePath); - const base64Data = imageBuffer.toString("base64"); - const ext = path.extname(imagePath).toLowerCase(); - const mimeTypeMap: Record = { - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".png": "image/png", - ".gif": "image/gif", - ".webp": "image/webp", - }; - const mediaType = mimeTypeMap[ext] || "image/png"; - - contentBlocks.push({ - type: "image", - source: { - type: "base64", - media_type: mediaType, - data: base64Data, - }, - }); - } catch (error) { - console.error(`[AgentService] Failed to load image ${imagePath}:`, error); - } - } - - if (contentBlocks.length > 1 || contentBlocks[0]?.type === "image") { - promptContent = contentBlocks; - } else { - promptContent = enhancedMessage; - } - } + // Build prompt content with images + const { content: promptContent } = await buildPromptWithImages( + message, + imagePaths, + undefined, // no workDir for agent service + true // include image paths in text + ); // Set the prompt in options options.prompt = promptContent; @@ -335,7 +284,7 @@ export class AgentService { message: currentAssistantMessage, }; } catch (error) { - if (error instanceof AbortError || (error as Error)?.name === "AbortError") { + if (isAbortError(error)) { session.isRunning = false; session.abortController = null; return { success: false, aborted: true }; diff --git a/apps/server/src/services/auto-mode-service.ts b/apps/server/src/services/auto-mode-service.ts index aadca1b5..e38351e9 100644 --- a/apps/server/src/services/auto-mode-service.ts +++ b/apps/server/src/services/auto-mode-service.ts @@ -17,16 +17,12 @@ import { promisify } from "util"; import path from "path"; import fs from "fs/promises"; import type { EventEmitter, EventType } from "../lib/events.js"; +import { buildPromptWithImages } from "../lib/prompt-builder.js"; +import { resolveModelString, DEFAULT_MODELS } from "../lib/model-resolver.js"; +import { isAbortError, classifyError } from "../lib/error-handler.js"; const execAsync = promisify(exec); -// Model name mappings for Claude (matching electron version) -const MODEL_MAP: Record = { - haiku: "claude-haiku-4-5", - sonnet: "claude-sonnet-4-20250514", - opus: "claude-opus-4-5-20251101", -}; - interface Feature { id: string; title: string; @@ -38,36 +34,6 @@ interface Feature { imagePaths?: Array; } -/** - * Get model string from feature's model property - * Supports model keys like "opus", "sonnet", "haiku" or full model strings - * Also supports OpenAI/Codex models like "gpt-5.2", "gpt-5.1-codex", etc. - */ -function getModelString(feature: Feature): string { - const modelKey = feature.model || "opus"; // Default to opus - - // Check if it's an OpenAI/Codex model (starts with "gpt-" or "o" for O-series) - if (modelKey.startsWith("gpt-") || modelKey.startsWith("o")) { - console.log(`[AutoMode] Using OpenAI/Codex model from feature ${feature.id}: ${modelKey} (passing through)`); - return modelKey; - } - - // If it's already a full Claude model string (contains "claude-"), use it directly - if (modelKey.includes("claude-")) { - console.log(`[AutoMode] Using Claude model from feature ${feature.id}: ${modelKey} (full model string)`); - return modelKey; - } - - // Otherwise, look it up in the Claude model map - const modelString = MODEL_MAP[modelKey] || MODEL_MAP.opus; - if (modelString !== MODEL_MAP.opus || modelKey === "opus") { - console.log(`[AutoMode] Resolved Claude model for feature ${feature.id}: "${modelKey}" -> "${modelString}"`); - } else { - console.warn(`[AutoMode] Unknown model key "${modelKey}" for feature ${feature.id}, defaulting to "${modelString}"`); - } - return modelString; -} - interface RunningFeature { featureId: string; projectPath: string; @@ -246,7 +212,7 @@ export class AutoModeService { ); // Get model from feature - const model = getModelString(feature); + const model = resolveModelString(feature.model, DEFAULT_MODELS.claude); console.log(`[AutoMode] Executing feature ${featureId} with model: ${model}`); // Run the agent with the feature's model and images @@ -262,7 +228,9 @@ export class AutoModeService { projectPath, }); } catch (error) { - if (error instanceof AbortError || (error as Error)?.name === "AbortError") { + const errorInfo = classifyError(error); + + if (errorInfo.isAbort) { this.emitAutoModeEvent("auto_mode_feature_complete", { featureId, passes: false, @@ -270,17 +238,12 @@ export class AutoModeService { projectPath, }); } else { - const errorMessage = (error as Error).message || "Unknown error"; - const isAuthError = errorMessage.includes("Authentication failed") || - errorMessage.includes("Invalid API key") || - errorMessage.includes("authentication_failed"); - console.error(`[AutoMode] Feature ${featureId} failed:`, error); await this.updateFeatureStatus(projectPath, featureId, "backlog"); this.emitAutoModeEvent("auto_mode_error", { featureId, - error: errorMessage, - errorType: isAuthError ? "authentication" : "execution", + error: errorInfo.message, + errorType: errorInfo.isAuth ? "authentication" : "execution", projectPath, }); } @@ -382,7 +345,7 @@ export class AutoModeService { try { // Load feature to get its model const feature = await this.loadFeature(projectPath, featureId); - const model = feature ? getModelString(feature) : MODEL_MAP.opus; + const model = resolveModelString(feature?.model, DEFAULT_MODELS.claude); console.log(`[AutoMode] Follow-up for feature ${featureId} using model: ${model}`); // Update feature status to in_progress @@ -513,7 +476,7 @@ Please continue from where you left off and address the new instructions above.` projectPath, }); } catch (error) { - if (!(error instanceof AbortError)) { + if (!isAbortError(error)) { this.emitAutoModeEvent("auto_mode_error", { featureId, error: (error as Error).message, @@ -909,7 +872,7 @@ When done, summarize what you implemented and any notes for the developer.`; imagePaths?: string[], model?: string ): Promise { - const finalModel = model || MODEL_MAP.opus; + const finalModel = resolveModelString(model, DEFAULT_MODELS.claude); console.log(`[AutoMode] runAgent called for feature ${featureId} with model: ${finalModel}`); // Get provider for this model @@ -919,51 +882,13 @@ When done, summarize what you implemented and any notes for the developer.`; `[AutoMode] Using provider "${provider.getName()}" for model "${finalModel}"` ); - // Build prompt content with images (like AgentService) - let promptContent: string | Array<{ type: string; text?: string; source?: object }> = prompt; - - if (imagePaths && imagePaths.length > 0) { - const contentBlocks: Array<{ type: string; text?: string; source?: object }> = []; - - // Add text block first - contentBlocks.push({ type: "text", text: prompt }); - - // Add image blocks (for vision models) - for (const imagePath of imagePaths) { - try { - // Make path absolute by prepending workDir if it's relative - const absolutePath = path.isAbsolute(imagePath) - ? imagePath - : path.join(workDir, imagePath); - - const imageBuffer = await fs.readFile(absolutePath); - const base64Data = imageBuffer.toString("base64"); - const ext = path.extname(imagePath).toLowerCase(); - const mimeTypeMap: Record = { - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".png": "image/png", - ".gif": "image/gif", - ".webp": "image/webp", - }; - const mediaType = mimeTypeMap[ext] || "image/png"; - - contentBlocks.push({ - type: "image", - source: { - type: "base64", - media_type: mediaType, - data: base64Data, - }, - }); - - } catch (error) { - console.error(`[AutoMode] Failed to load image ${imagePath}:`, error); - } - } - - promptContent = contentBlocks; - } + // Build prompt content with images using utility + const { content: promptContent } = await buildPromptWithImages( + prompt, + imagePaths, + workDir, + false // don't duplicate paths in text + ); const options: ExecuteOptions = { prompt: promptContent, diff --git a/docs/server/providers.md b/docs/server/providers.md new file mode 100644 index 00000000..ed591bda --- /dev/null +++ b/docs/server/providers.md @@ -0,0 +1,786 @@ +# Provider Architecture Reference + +This document describes the modular provider architecture in `apps/server/src/providers/` that enables support for multiple AI model providers (Claude SDK, OpenAI Codex CLI, and future providers like Cursor, OpenCode, etc.). + +--- + +## Table of Contents + +1. [Architecture Overview](#architecture-overview) +2. [Provider Interface](#provider-interface) +3. [Available Providers](#available-providers) +4. [Provider Factory](#provider-factory) +5. [Adding New Providers](#adding-new-providers) +6. [Provider Types](#provider-types) +7. [Best Practices](#best-practices) + +--- + +## Architecture Overview + +The provider architecture separates AI model execution logic from business logic, enabling clean abstraction and easy extensibility. + +### Architecture Diagram + +``` +┌─────────────────────────────────────────┐ +│ AgentService / AutoModeService │ +│ (No provider logic) │ +└──────────────────┬──────────────────────┘ + │ + ┌─────────▼──────────┐ + │ ProviderFactory │ Model-based routing + │ (Routes by model) │ "gpt-*" → Codex + └─────────┬──────────┘ "claude-*" → Claude + │ + ┌────────────┴────────────┐ + │ │ +┌─────▼──────┐ ┌──────▼──────┐ +│ Claude │ │ Codex │ +│ Provider │ │ Provider │ +│ (Agent SDK)│ │ (CLI Spawn) │ +└────────────┘ └─────────────┘ +``` + +### Key Benefits + +- ✅ **Adding new providers**: Only 1 new file + 1 line in factory +- ✅ **Services remain clean**: No provider-specific logic +- ✅ **All providers implement same interface**: Consistent behavior +- ✅ **Model prefix determines provider**: Automatic routing +- ✅ **Easy to test**: Each provider can be tested independently + +--- + +## Provider Interface + +**Location**: `apps/server/src/providers/base-provider.ts` + +All providers must extend `BaseProvider` and implement the required methods. + +### BaseProvider Abstract Class + +```typescript +export abstract class BaseProvider { + protected config: ProviderConfig; + + constructor(config: ProviderConfig = {}) { + this.config = config; + } + + /** + * Get provider name (e.g., "claude", "codex") + */ + abstract getName(): string; + + /** + * Execute a query and stream responses + */ + abstract executeQuery(options: ExecuteOptions): AsyncGenerator; + + /** + * Detect provider installation status + */ + abstract detectInstallation(): Promise; + + /** + * Get available models for this provider + */ + abstract getAvailableModels(): ModelDefinition[]; + + /** + * Check if provider supports a specific feature (optional) + */ + supportsFeature(feature: string): boolean { + return false; + } +} +``` + +### Shared Types + +**Location**: `apps/server/src/providers/types.ts` + +#### ExecuteOptions + +Input configuration for executing queries: + +```typescript +export interface ExecuteOptions { + prompt: string | Array<{ type: string; text?: string; source?: object }>; + model: string; + cwd: string; + systemPrompt?: string; + maxTurns?: number; + allowedTools?: string[]; + mcpServers?: Record; + abortController?: AbortController; + conversationHistory?: ConversationMessage[]; +} +``` + +#### ProviderMessage + +Output messages streamed from providers: + +```typescript +export interface ProviderMessage { + type: "assistant" | "user" | "error" | "result"; + subtype?: "success" | "error"; + message?: { + role: "user" | "assistant"; + content: ContentBlock[]; + }; + result?: string; + error?: string; +} +``` + +#### ContentBlock + +Individual content blocks in messages: + +```typescript +export interface ContentBlock { + type: "text" | "tool_use" | "thinking" | "tool_result"; + text?: string; + thinking?: string; + name?: string; + input?: unknown; + tool_use_id?: string; + content?: string; +} +``` + +--- + +## Available Providers + +### 1. Claude Provider (SDK-based) + +**Location**: `apps/server/src/providers/claude-provider.ts` + +Uses `@anthropic-ai/claude-agent-sdk` for direct SDK integration. + +#### Features + +- ✅ Native multi-turn conversation support +- ✅ Vision support (images) +- ✅ Tool use (Read, Write, Edit, Glob, Grep, Bash, WebSearch, WebFetch) +- ✅ Thinking blocks (extended thinking) +- ✅ Streaming responses +- ✅ No CLI installation required (npm dependency) + +#### Model Detection + +Routes models that: +- Start with `"claude-"` (e.g., `"claude-opus-4-5-20251101"`) +- Are Claude aliases: `"opus"`, `"sonnet"`, `"haiku"` + +#### Authentication + +Requires one of: +- `ANTHROPIC_API_KEY` environment variable +- `CLAUDE_CODE_OAUTH_TOKEN` environment variable + +#### Example Usage + +```typescript +const provider = new ClaudeProvider(); + +const stream = provider.executeQuery({ + prompt: "What is 2+2?", + model: "claude-opus-4-5-20251101", + cwd: "/project/path", + systemPrompt: "You are a helpful assistant.", + maxTurns: 20, + allowedTools: ["Read", "Write", "Bash"], + abortController: new AbortController(), + conversationHistory: [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi! How can I help?" } + ] +}); + +for await (const msg of stream) { + if (msg.type === "assistant") { + console.log(msg.message?.content); + } +} +``` + +#### Conversation History Handling + +Uses `convertHistoryToMessages()` utility to convert history to SDK format: + +```typescript +const historyMessages = convertHistoryToMessages(conversationHistory); +for (const msg of historyMessages) { + yield msg; // Yield to SDK +} +``` + +--- + +### 2. Codex Provider (CLI-based) + +**Location**: `apps/server/src/providers/codex-provider.ts` + +Spawns OpenAI Codex CLI as a subprocess and converts JSONL output to provider format. + +#### Features + +- ✅ Subprocess execution (`codex exec --model --json --full-auto`) +- ✅ JSONL stream parsing +- ✅ Supports GPT-5.1/5.2 Codex models +- ✅ Vision support (GPT-5.1, GPT-5.2) +- ✅ Tool use via MCP servers +- ✅ Timeout detection (30s no output) +- ✅ Abort signal handling + +#### Model Detection + +Routes models that: +- Start with `"gpt-"` (e.g., `"gpt-5.2"`, `"gpt-5.1-codex-max"`) +- Start with `"o"` (e.g., `"o1"`, `"o1-mini"`) + +#### Available Models + +| Model | Description | Context | Max Output | Vision | +|-------|-------------|---------|------------|--------| +| `gpt-5.2` | Latest Codex model | 256K | 32K | Yes | +| `gpt-5.1-codex-max` | Maximum capability | 256K | 32K | Yes | +| `gpt-5.1-codex` | Standard Codex | 256K | 32K | Yes | +| `gpt-5.1-codex-mini` | Lightweight | 256K | 16K | No | +| `gpt-5.1` | General-purpose | 256K | 32K | Yes | + +#### Authentication + +Supports two methods: +1. **CLI login**: `codex login` (OAuth tokens stored in `~/.codex/auth.json`) +2. **API key**: `OPENAI_API_KEY` environment variable + +#### Installation Detection + +Uses `CodexCliDetector` to check: +- PATH for `codex` command +- npm global: `npm list -g @openai/codex` +- Homebrew (macOS): `/opt/homebrew/bin/codex` +- Common paths: `~/.local/bin/codex`, `/usr/local/bin/codex` + +#### Example Usage + +```typescript +const provider = new CodexProvider(); + +const stream = provider.executeQuery({ + prompt: "Fix the bug in main.ts", + model: "gpt-5.2", + cwd: "/project/path", + systemPrompt: "You are an expert TypeScript developer.", + abortController: new AbortController() +}); + +for await (const msg of stream) { + if (msg.type === "assistant") { + console.log(msg.message?.content); + } else if (msg.type === "error") { + console.error(msg.error); + } +} +``` + +#### JSONL Event Conversion + +Codex CLI outputs JSONL events that get converted to `ProviderMessage` format: + +| Codex Event | Provider Message | +|-------------|------------------| +| `item.completed` (reasoning) | `{ type: "assistant", content: [{ type: "thinking" }] }` | +| `item.completed` (agent_message) | `{ type: "assistant", content: [{ type: "text" }] }` | +| `item.completed` (command_execution) | `{ type: "assistant", content: [{ type: "text", text: "```bash\n...\n```" }] }` | +| `item.started` (command_execution) | `{ type: "assistant", content: [{ type: "tool_use" }] }` | +| `item.updated` (todo_list) | `{ type: "assistant", content: [{ type: "text", text: "**Updated Todo List:**..." }] }` | +| `thread.completed` | `{ type: "result", subtype: "success" }` | +| `error` | `{ type: "error", error: "..." }` | + +#### Conversation History Handling + +Uses `formatHistoryAsText()` utility to prepend history as text context (CLI doesn't support native multi-turn): + +```typescript +const historyText = formatHistoryAsText(conversationHistory); +combinedPrompt = `${historyText}Current request:\n${combinedPrompt}`; +``` + +#### MCP Server Configuration + +**Location**: `apps/server/src/providers/codex-config-manager.ts` + +Manages TOML configuration for MCP servers: + +```typescript +await codexConfigManager.configureMcpServer(cwd, mcpServerScriptPath); +``` + +Generates `.codex/config.toml`: +```toml +[mcp_servers.automaker-tools] +command = "node" +args = ["/path/to/mcp-server.js"] +enabled_tools = ["UpdateFeatureStatus"] +``` + +--- + +## Provider Factory + +**Location**: `apps/server/src/providers/provider-factory.ts` + +Routes requests to the appropriate provider based on model string. + +### Model-Based Routing + +```typescript +export class ProviderFactory { + /** + * Get provider for a specific model + */ + static getProviderForModel(modelId: string): BaseProvider { + const lowerModel = modelId.toLowerCase(); + + // OpenAI/Codex models + if (lowerModel.startsWith("gpt-") || lowerModel.startsWith("o")) { + return new CodexProvider(); + } + + // Claude models + if (lowerModel.startsWith("claude-") || + ["haiku", "sonnet", "opus"].includes(lowerModel)) { + return new ClaudeProvider(); + } + + // Default to Claude + return new ClaudeProvider(); + } + + /** + * Check installation status of all providers + */ + static async checkAllProviders(): Promise> { + const claude = new ClaudeProvider(); + const codex = new CodexProvider(); + + return { + claude: await claude.detectInstallation(), + codex: await codex.detectInstallation(), + }; + } +} +``` + +### Usage in Services + +```typescript +import { ProviderFactory } from "../providers/provider-factory.js"; + +// In AgentService or AutoModeService +const provider = ProviderFactory.getProviderForModel(model); +const stream = provider.executeQuery(options); + +for await (const msg of stream) { + // Handle messages (format is consistent across all providers) +} +``` + +--- + +## Adding New Providers + +### Step 1: Create Provider File + +Create `apps/server/src/providers/[name]-provider.ts`: + +```typescript +import { BaseProvider } from "./base-provider.js"; +import type { + ExecuteOptions, + ProviderMessage, + InstallationStatus, + ModelDefinition, +} from "./types.js"; + +export class CursorProvider extends BaseProvider { + getName(): string { + return "cursor"; + } + + async *executeQuery(options: ExecuteOptions): AsyncGenerator { + // Implementation here + // 1. Spawn cursor CLI or use SDK + // 2. Convert output to ProviderMessage format + // 3. Yield messages + } + + async detectInstallation(): Promise { + // Check if cursor is installed + // Return { installed: boolean, path?: string, version?: string } + } + + getAvailableModels(): ModelDefinition[] { + return [ + { + id: "cursor-premium", + name: "Cursor Premium", + modelString: "cursor-premium", + provider: "cursor", + description: "Cursor's premium model", + contextWindow: 200000, + maxOutputTokens: 8192, + supportsVision: true, + supportsTools: true, + tier: "premium", + default: true, + } + ]; + } + + supportsFeature(feature: string): boolean { + const supportedFeatures = ["tools", "text", "vision"]; + return supportedFeatures.includes(feature); + } +} +``` + +### Step 2: Add Routing in Factory + +Update `apps/server/src/providers/provider-factory.ts`: + +```typescript +import { CursorProvider } from "./cursor-provider.js"; + +static getProviderForModel(modelId: string): BaseProvider { + const lowerModel = modelId.toLowerCase(); + + // Cursor models + if (lowerModel.startsWith("cursor-")) { + return new CursorProvider(); + } + + // ... existing routing +} + +static async checkAllProviders() { + const cursor = new CursorProvider(); + + return { + claude: await claude.detectInstallation(), + codex: await codex.detectInstallation(), + cursor: await cursor.detectInstallation(), // NEW + }; +} +``` + +### Step 3: Update Models List + +Update `apps/server/src/routes/models.ts`: + +```typescript +{ + id: "cursor-premium", + name: "Cursor Premium", + provider: "cursor", + contextWindow: 200000, + maxOutputTokens: 8192, + supportsVision: true, + supportsTools: true, +} +``` + +### Step 4: Done! + +No changes needed in: +- ✅ AgentService +- ✅ AutoModeService +- ✅ Any business logic + +The provider architecture handles everything automatically. + +--- + +## Provider Types + +### SDK-Based Providers (like Claude) + +**Characteristics**: +- Direct SDK/library integration +- No subprocess spawning +- Native multi-turn support +- Streaming via async generators + +**Example**: ClaudeProvider using `@anthropic-ai/claude-agent-sdk` + +**Advantages**: +- Lower latency +- More control over options +- Easier error handling +- No CLI installation required + +--- + +### CLI-Based Providers (like Codex) + +**Characteristics**: +- Subprocess spawning +- JSONL stream parsing +- Text-based conversation history +- Requires CLI installation + +**Example**: CodexProvider using `codex exec --json` + +**Advantages**: +- Access to CLI-only features +- No SDK dependency +- Can use any CLI tool + +**Implementation Pattern**: +1. Use `spawnJSONLProcess()` from `subprocess-manager.ts` +2. Convert JSONL events to `ProviderMessage` format +3. Handle authentication (CLI login or API key) +4. Implement timeout detection + +--- + +## Best Practices + +### 1. Message Format Consistency + +All providers MUST output the same `ProviderMessage` format so services can handle them uniformly: + +```typescript +// ✅ Correct - Consistent format +yield { + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "Response" }] + } +}; + +// ❌ Incorrect - Provider-specific format +yield { + customType: "response", + data: "Response" +}; +``` + +### 2. Error Handling + +Always yield error messages, never throw: + +```typescript +// ✅ Correct +try { + // ... +} catch (error) { + yield { + type: "error", + error: (error as Error).message + }; + return; +} + +// ❌ Incorrect +throw new Error("Provider failed"); +``` + +### 3. Abort Signal Support + +Respect the abort controller: + +```typescript +if (abortController?.signal.aborted) { + yield { type: "error", error: "Operation cancelled" }; + return; +} +``` + +### 4. Conversation History + +- **SDK providers**: Use `convertHistoryToMessages()` and yield messages +- **CLI providers**: Use `formatHistoryAsText()` and prepend to prompt + +### 5. Image Handling + +- **Vision models**: Pass images as content blocks +- **Non-vision models**: Extract text only using utilities + +### 6. Logging + +Use consistent logging prefixes: + +```typescript +console.log(`[${this.getName()}Provider] Operation started`); +console.error(`[${this.getName()}Provider] Error:`, error); +``` + +### 7. Installation Detection + +Implement thorough detection: +- Check multiple installation methods +- Verify authentication +- Return detailed status + +### 8. Model Definitions + +Provide accurate model metadata: + +```typescript +{ + id: "model-id", + name: "Human-readable name", + modelString: "exact-model-string", + provider: "provider-name", + description: "What this model is good for", + contextWindow: 200000, + maxOutputTokens: 8192, + supportsVision: true, + supportsTools: true, + tier: "premium" | "standard" | "basic", + default: false +} +``` + +--- + +## Testing Providers + +### Unit Tests + +Test each provider method independently: + +```typescript +describe("ClaudeProvider", () => { + it("should detect installation", async () => { + const provider = new ClaudeProvider(); + const status = await provider.detectInstallation(); + + expect(status.installed).toBe(true); + expect(status.method).toBe("sdk"); + }); + + it("should stream messages correctly", async () => { + const provider = new ClaudeProvider(); + const messages = []; + + for await (const msg of provider.executeQuery(options)) { + messages.push(msg); + } + + expect(messages.length).toBeGreaterThan(0); + expect(messages[0].type).toBe("assistant"); + }); +}); +``` + +### Integration Tests + +Test provider interaction with services: + +```typescript +describe("Provider Integration", () => { + it("should work with AgentService", async () => { + const provider = ProviderFactory.getProviderForModel("claude-opus-4-5-20251101"); + + // Test full workflow + }); +}); +``` + +--- + +## Environment Variables + +### Claude Provider + +```bash +# Required (one of): +ANTHROPIC_API_KEY=sk-ant-... +CLAUDE_CODE_OAUTH_TOKEN=... +``` + +### Codex Provider + +```bash +# Required (one of): +OPENAI_API_KEY=sk-... +# OR run: codex login + +# Optional: +CODEX_CLI_PATH=/custom/path/to/codex +``` + +--- + +## Troubleshooting + +### Provider Not Found + +**Problem**: `ProviderFactory.getProviderForModel()` returns wrong provider + +**Solution**: Check model string prefix in factory routing + +### Authentication Errors + +**Problem**: Provider fails with auth error + +**Solution**: +1. Check environment variables +2. For CLI providers, verify CLI login status +3. Check `detectInstallation()` output + +### JSONL Parsing Errors (CLI providers) + +**Problem**: Failed to parse JSONL line + +**Solution**: +1. Check CLI output format +2. Verify JSON is valid +3. Add error handling for malformed lines + +### Timeout Issues (CLI providers) + +**Problem**: Subprocess hangs + +**Solution**: +1. Increase timeout in `spawnJSONLProcess` options +2. Check CLI process for hangs +3. Verify abort signal handling + +--- + +## Future Provider Ideas + +Potential providers to add: + +1. **Cursor Provider** (`cursor-*`) + - CLI-based + - Code completion specialist + +2. **OpenCode Provider** (`opencode-*`) + - SDK or CLI-based + - Open-source alternative + +3. **Gemini Provider** (`gemini-*`) + - Google's AI models + - SDK-based via `@google/generative-ai` + +4. **Ollama Provider** (`ollama-*`) + - Local model hosting + - CLI or HTTP API + +Each would follow the same pattern: +1. Create `[name]-provider.ts` implementing `BaseProvider` +2. Add routing in `provider-factory.ts` +3. Update models list +4. Done! ✅ diff --git a/docs/server/utilities.md b/docs/server/utilities.md new file mode 100644 index 00000000..d14aaf5f --- /dev/null +++ b/docs/server/utilities.md @@ -0,0 +1,672 @@ +# Server Utilities Reference + +This document describes all utility modules available in `apps/server/src/lib/`. These utilities provide reusable functionality for image handling, prompt building, model resolution, conversation management, and error handling. + +--- + +## Table of Contents + +1. [Image Handler](#image-handler) +2. [Prompt Builder](#prompt-builder) +3. [Model Resolver](#model-resolver) +4. [Conversation Utils](#conversation-utils) +5. [Error Handler](#error-handler) +6. [Subprocess Manager](#subprocess-manager) +7. [Events](#events) +8. [Auth](#auth) +9. [Security](#security) + +--- + +## Image Handler + +**Location**: `apps/server/src/lib/image-handler.ts` + +Centralized utilities for processing image files, including MIME type detection, base64 encoding, and content block generation for Claude SDK format. + +### Functions + +#### `getMimeTypeForImage(imagePath: string): string` + +Get MIME type for an image file based on its extension. + +**Supported formats**: +- `.jpg`, `.jpeg` → `image/jpeg` +- `.png` → `image/png` +- `.gif` → `image/gif` +- `.webp` → `image/webp` +- Default: `image/png` + +**Example**: +```typescript +import { getMimeTypeForImage } from "../lib/image-handler.js"; + +const mimeType = getMimeTypeForImage("/path/to/image.jpg"); +// Returns: "image/jpeg" +``` + +--- + +#### `readImageAsBase64(imagePath: string): Promise` + +Read an image file and convert to base64 with metadata. + +**Returns**: `ImageData` +```typescript +interface ImageData { + base64: string; // Base64-encoded image data + mimeType: string; // MIME type + filename: string; // File basename + originalPath: string; // Original file path +} +``` + +**Example**: +```typescript +const imageData = await readImageAsBase64("/path/to/photo.png"); +console.log(imageData.base64); // "iVBORw0KG..." +console.log(imageData.mimeType); // "image/png" +console.log(imageData.filename); // "photo.png" +``` + +--- + +#### `convertImagesToContentBlocks(imagePaths: string[], workDir?: string): Promise` + +Convert image paths to content blocks in Claude SDK format. Handles both relative and absolute paths. + +**Parameters**: +- `imagePaths` - Array of image file paths +- `workDir` - Optional working directory for resolving relative paths + +**Returns**: Array of `ImageContentBlock` +```typescript +interface ImageContentBlock { + type: "image"; + source: { + type: "base64"; + media_type: string; + data: string; + }; +} +``` + +**Example**: +```typescript +const imageBlocks = await convertImagesToContentBlocks( + ["./screenshot.png", "/absolute/path/diagram.jpg"], + "/project/root" +); + +// Use in prompt content +const contentBlocks = [ + { type: "text", text: "Analyze these images:" }, + ...imageBlocks +]; +``` + +--- + +#### `formatImagePathsForPrompt(imagePaths: string[]): string` + +Format image paths as a bulleted list for inclusion in text prompts. + +**Returns**: Formatted string with image paths, or empty string if no images. + +**Example**: +```typescript +const pathsList = formatImagePathsForPrompt([ + "/screenshots/login.png", + "/diagrams/architecture.png" +]); + +// Returns: +// "\n\nAttached images:\n- /screenshots/login.png\n- /diagrams/architecture.png\n" +``` + +--- + +## Prompt Builder + +**Location**: `apps/server/src/lib/prompt-builder.ts` + +Standardized prompt building that combines text prompts with image attachments. + +### Functions + +#### `buildPromptWithImages(basePrompt: string, imagePaths?: string[], workDir?: string, includeImagePaths: boolean = false): Promise` + +Build a prompt with optional image attachments. + +**Parameters**: +- `basePrompt` - The text prompt +- `imagePaths` - Optional array of image file paths +- `workDir` - Optional working directory for resolving relative paths +- `includeImagePaths` - Whether to append image paths to the text (default: false) + +**Returns**: `PromptWithImages` +```typescript +interface PromptWithImages { + content: PromptContent; // string | Array + hasImages: boolean; +} + +type PromptContent = string | Array<{ + type: string; + text?: string; + source?: object; +}>; +``` + +**Example**: +```typescript +import { buildPromptWithImages } from "../lib/prompt-builder.js"; + +// Without images +const { content } = await buildPromptWithImages("What is 2+2?"); +// content: "What is 2+2?" (simple string) + +// With images +const { content, hasImages } = await buildPromptWithImages( + "Analyze this screenshot", + ["/path/to/screenshot.png"], + "/project/root", + true // include image paths in text +); +// content: [ +// { type: "text", text: "Analyze this screenshot\n\nAttached images:\n- /path/to/screenshot.png\n" }, +// { type: "image", source: { type: "base64", media_type: "image/png", data: "..." } } +// ] +// hasImages: true +``` + +**Use Cases**: +- **AgentService**: Set `includeImagePaths: true` to list paths for Read tool access +- **AutoModeService**: Set `includeImagePaths: false` to avoid duplication in feature descriptions + +--- + +## Model Resolver + +**Location**: `apps/server/src/lib/model-resolver.ts` + +Centralized model string mapping and resolution for handling model aliases and provider detection. + +### Constants + +#### `CLAUDE_MODEL_MAP` + +Model alias mapping for Claude models. + +```typescript +export const CLAUDE_MODEL_MAP: Record = { + haiku: "claude-haiku-4-5", + sonnet: "claude-sonnet-4-20250514", + opus: "claude-opus-4-5-20251101", +} as const; +``` + +#### `DEFAULT_MODELS` + +Default models per provider. + +```typescript +export const DEFAULT_MODELS = { + claude: "claude-opus-4-5-20251101", + openai: "gpt-5.2", +} as const; +``` + +### Functions + +#### `resolveModelString(modelKey?: string, defaultModel: string = DEFAULT_MODELS.claude): string` + +Resolve a model key/alias to a full model string. + +**Logic**: +1. If `modelKey` is undefined → return `defaultModel` +2. If starts with `"gpt-"` or `"o"` → pass through (OpenAI/Codex model) +3. If includes `"claude-"` → pass through (full Claude model string) +4. If in `CLAUDE_MODEL_MAP` → return mapped value +5. Otherwise → return `defaultModel` with warning + +**Example**: +```typescript +import { resolveModelString, DEFAULT_MODELS } from "../lib/model-resolver.js"; + +resolveModelString("opus"); +// Returns: "claude-opus-4-5-20251101" +// Logs: "[ModelResolver] Resolved model alias: "opus" -> "claude-opus-4-5-20251101"" + +resolveModelString("gpt-5.2"); +// Returns: "gpt-5.2" +// Logs: "[ModelResolver] Using OpenAI/Codex model: gpt-5.2" + +resolveModelString("claude-sonnet-4-20250514"); +// Returns: "claude-sonnet-4-20250514" +// Logs: "[ModelResolver] Using full Claude model string: claude-sonnet-4-20250514" + +resolveModelString("invalid-model"); +// Returns: "claude-opus-4-5-20251101" +// Logs: "[ModelResolver] Unknown model key "invalid-model", using default: "claude-opus-4-5-20251101"" +``` + +--- + +#### `getEffectiveModel(explicitModel?: string, sessionModel?: string, defaultModel?: string): string` + +Get the effective model from multiple sources with priority. + +**Priority**: explicit model > session model > default model + +**Example**: +```typescript +import { getEffectiveModel } from "../lib/model-resolver.js"; + +// Explicit model takes precedence +getEffectiveModel("sonnet", "opus"); +// Returns: "claude-sonnet-4-20250514" + +// Falls back to session model +getEffectiveModel(undefined, "haiku"); +// Returns: "claude-haiku-4-5" + +// Falls back to default +getEffectiveModel(undefined, undefined, "gpt-5.2"); +// Returns: "gpt-5.2" +``` + +--- + +## Conversation Utils + +**Location**: `apps/server/src/lib/conversation-utils.ts` + +Standardized conversation history processing for both SDK-based and CLI-based providers. + +### Types + +```typescript +import type { ConversationMessage } from "../providers/types.js"; + +interface ConversationMessage { + role: "user" | "assistant"; + content: string | Array<{ type: string; text?: string; source?: object }>; +} +``` + +### Functions + +#### `extractTextFromContent(content: string | Array): string` + +Extract plain text from message content (handles both string and array formats). + +**Example**: +```typescript +import { extractTextFromContent } from "../lib/conversation-utils.js"; + +// String content +extractTextFromContent("Hello world"); +// Returns: "Hello world" + +// Array content +extractTextFromContent([ + { type: "text", text: "Hello" }, + { type: "image", source: {...} }, + { type: "text", text: "world" } +]); +// Returns: "Hello\nworld" +``` + +--- + +#### `normalizeContentBlocks(content: string | Array): Array` + +Normalize message content to array format. + +**Example**: +```typescript +// String → array +normalizeContentBlocks("Hello"); +// Returns: [{ type: "text", text: "Hello" }] + +// Array → pass through +normalizeContentBlocks([{ type: "text", text: "Hello" }]); +// Returns: [{ type: "text", text: "Hello" }] +``` + +--- + +#### `formatHistoryAsText(history: ConversationMessage[]): string` + +Format conversation history as plain text for CLI-based providers (e.g., Codex). + +**Returns**: Formatted text with role labels, or empty string if no history. + +**Example**: +```typescript +const history = [ + { role: "user", content: "What is 2+2?" }, + { role: "assistant", content: "2+2 equals 4." } +]; + +const formatted = formatHistoryAsText(history); +// Returns: +// "Previous conversation: +// +// User: What is 2+2? +// +// Assistant: 2+2 equals 4. +// +// --- +// +// " +``` + +--- + +#### `convertHistoryToMessages(history: ConversationMessage[]): Array` + +Convert conversation history to Claude SDK message format. + +**Returns**: Array of SDK-formatted messages ready to yield in async generator. + +**Example**: +```typescript +const history = [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" } +]; + +const messages = convertHistoryToMessages(history); +// Returns: +// [ +// { +// type: "user", +// session_id: "", +// message: { +// role: "user", +// content: [{ type: "text", text: "Hello" }] +// }, +// parent_tool_use_id: null +// }, +// { +// type: "assistant", +// session_id: "", +// message: { +// role: "assistant", +// content: [{ type: "text", text: "Hi there!" }] +// }, +// parent_tool_use_id: null +// } +// ] +``` + +--- + +## Error Handler + +**Location**: `apps/server/src/lib/error-handler.ts` + +Standardized error classification and handling utilities. + +### Types + +```typescript +export type ErrorType = "authentication" | "abort" | "execution" | "unknown"; + +export interface ErrorInfo { + type: ErrorType; + message: string; + isAbort: boolean; + isAuth: boolean; + originalError: unknown; +} +``` + +### Functions + +#### `isAbortError(error: unknown): boolean` + +Check if an error is an abort/cancellation error. + +**Example**: +```typescript +import { isAbortError } from "../lib/error-handler.js"; + +try { + // ... operation +} catch (error) { + if (isAbortError(error)) { + console.log("Operation was cancelled"); + return { success: false, aborted: true }; + } +} +``` + +--- + +#### `isAuthenticationError(errorMessage: string): boolean` + +Check if an error is an authentication/API key error. + +**Detects**: +- "Authentication failed" +- "Invalid API key" +- "authentication_failed" +- "Fix external API key" + +**Example**: +```typescript +if (isAuthenticationError(error.message)) { + console.error("Please check your API key configuration"); +} +``` + +--- + +#### `classifyError(error: unknown): ErrorInfo` + +Classify an error into a specific type. + +**Example**: +```typescript +import { classifyError } from "../lib/error-handler.js"; + +try { + // ... operation +} catch (error) { + const errorInfo = classifyError(error); + + switch (errorInfo.type) { + case "authentication": + // Handle auth errors + break; + case "abort": + // Handle cancellation + break; + case "execution": + // Handle other errors + break; + } +} +``` + +--- + +#### `getUserFriendlyErrorMessage(error: unknown): string` + +Get a user-friendly error message. + +**Example**: +```typescript +try { + // ... operation +} catch (error) { + const friendlyMessage = getUserFriendlyErrorMessage(error); + // "Operation was cancelled" for abort errors + // "Authentication failed. Please check your API key." for auth errors + // Original error message for other errors +} +``` + +--- + +## Subprocess Manager + +**Location**: `apps/server/src/lib/subprocess-manager.ts` + +Utilities for spawning CLI processes and parsing JSONL streams (used by Codex provider). + +### Types + +```typescript +export interface SubprocessOptions { + command: string; + args: string[]; + cwd: string; + env?: Record; + abortController?: AbortController; + timeout?: number; // Milliseconds of no output before timeout +} + +export interface SubprocessResult { + stdout: string; + stderr: string; + exitCode: number | null; +} +``` + +### Functions + +#### `async function* spawnJSONLProcess(options: SubprocessOptions): AsyncGenerator` + +Spawns a subprocess and streams JSONL output line-by-line. + +**Features**: +- Parses each line as JSON +- Handles abort signals +- 30-second timeout detection for hanging processes +- Collects stderr for error reporting +- Continues processing other lines if one fails to parse + +**Example**: +```typescript +import { spawnJSONLProcess } from "../lib/subprocess-manager.js"; + +const stream = spawnJSONLProcess({ + command: "codex", + args: ["exec", "--model", "gpt-5.2", "--json", "--full-auto", "Fix the bug"], + cwd: "/project/path", + env: { OPENAI_API_KEY: "sk-..." }, + abortController: new AbortController(), + timeout: 30000 +}); + +for await (const event of stream) { + console.log("Received event:", event); + // Process JSONL events +} +``` + +--- + +#### `async function spawnProcess(options: SubprocessOptions): Promise` + +Spawns a subprocess and collects all output. + +**Example**: +```typescript +const result = await spawnProcess({ + command: "git", + args: ["status"], + cwd: "/project/path" +}); + +console.log(result.stdout); // Git status output +console.log(result.exitCode); // 0 for success +``` + +--- + +## Events + +**Location**: `apps/server/src/lib/events.ts` + +Event emitter system for WebSocket communication. + +**Documented separately** - see existing codebase for event types and usage. + +--- + +## Auth + +**Location**: `apps/server/src/lib/auth.ts` + +Authentication utilities for API endpoints. + +**Documented separately** - see existing codebase for authentication flow. + +--- + +## Security + +**Location**: `apps/server/src/lib/security.ts` + +Security utilities for input validation and sanitization. + +**Documented separately** - see existing codebase for security patterns. + +--- + +## Best Practices + +### When to Use Which Utility + +1. **Image handling** → Always use `image-handler.ts` utilities + - ✅ Do: `convertImagesToContentBlocks(imagePaths, workDir)` + - ❌ Don't: Manually read files and encode base64 + +2. **Prompt building** → Use `prompt-builder.ts` for consistency + - ✅ Do: `buildPromptWithImages(text, images, workDir, includePathsInText)` + - ❌ Don't: Manually construct content block arrays + +3. **Model resolution** → Use `model-resolver.ts` for all model handling + - ✅ Do: `resolveModelString(feature.model, DEFAULT_MODELS.claude)` + - ❌ Don't: Inline model mapping logic + +4. **Error handling** → Use `error-handler.ts` for classification + - ✅ Do: `if (isAbortError(error)) { ... }` + - ❌ Don't: `if (error instanceof AbortError || error.name === "AbortError") { ... }` + +### Importing Utilities + +Always use `.js` extension in imports for ESM compatibility: + +```typescript +// ✅ Correct +import { buildPromptWithImages } from "../lib/prompt-builder.js"; + +// ❌ Incorrect +import { buildPromptWithImages } from "../lib/prompt-builder"; +``` + +--- + +## Testing Utilities + +When writing tests for utilities: + +1. **Unit tests** - Test each function in isolation +2. **Integration tests** - Test utilities working together +3. **Mock external dependencies** - File system, child processes + +Example: +```typescript +describe("image-handler", () => { + it("should detect MIME type correctly", () => { + expect(getMimeTypeForImage("photo.jpg")).toBe("image/jpeg"); + expect(getMimeTypeForImage("diagram.png")).toBe("image/png"); + }); +}); +```