chore: update CI configuration and enhance test stability

- Added deterministic API key and environment variables in e2e-tests.yml to ensure consistent test behavior.
- Refactored CodexProvider tests to improve type safety and mock handling, ensuring reliable test execution.
- Updated provider-factory tests to mock installation detection for CodexProvider, enhancing test isolation.
- Adjusted Playwright configuration to conditionally use external backend, improving flexibility in test environments.
- Enhanced kill-test-servers script to handle external server scenarios, ensuring proper cleanup of test processes.

These changes improve the reliability and maintainability of the testing framework, leading to a more stable development experience.

apps/ui/tests/utils/api/client.ts

@@ -282,28 +282,40 @@ export async function apiListBranches(
  */
 export async function authenticateWithApiKey(page: Page, apiKey: string): Promise<boolean> {
   try {
+    // Ensure the backend is up before attempting login (especially in local runs where
+    // the backend may be started separately from Playwright).
+    const start = Date.now();
+    while (Date.now() - start < 15000) {
+      try {
+        const health = await page.request.get(`${API_BASE_URL}/api/health`, {
+          timeout: 3000,
+        });
+        if (health.ok()) break;
+      } catch {
+        // Retry
+      }
+      await page.waitForTimeout(250);
+    }
+
     // Ensure we're on a page (needed for cookies to work)
     const currentUrl = page.url();
     if (!currentUrl || currentUrl === 'about:blank') {
       await page.goto('http://localhost:3007', { waitUntil: 'domcontentloaded' });
     }
 
-    // Use browser context fetch to ensure cookies are set in the browser
-    const response = await page.evaluate(
-      async ({ url, apiKey }) => {
-        const res = await fetch(url, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          credentials: 'include',
-          body: JSON.stringify({ apiKey }),
-        });
-        const data = await res.json();
-        return { success: data.success, token: data.token };
-      },
-      { url: `${API_BASE_URL}/api/auth/login`, apiKey }
-    );
+    // Use Playwright request API (tied to this browser context) to avoid flakiness
+    // with cross-origin fetch inside page.evaluate.
+    const loginResponse = await page.request.post(`${API_BASE_URL}/api/auth/login`, {
+      data: { apiKey },
+      headers: { 'Content-Type': 'application/json' },
+      timeout: 15000,
+    });
+    const response = (await loginResponse.json().catch(() => null)) as {
+      success?: boolean;
+      token?: string;
+    } | null;
 
-    if (response.success && response.token) {
+    if (response?.success && response.token) {
       // Manually set the cookie in the browser context
       // The server sets a cookie named 'automaker_session' (see SESSION_COOKIE_NAME in auth.ts)
       await page.context().addCookies([
@@ -322,22 +334,19 @@ export async function authenticateWithApiKey(page: Page, apiKey: string): Promis
       let attempts = 0;
       const maxAttempts = 10;
       while (attempts < maxAttempts) {
-        const statusResponse = await page.evaluate(
-          async ({ url }) => {
-            const res = await fetch(url, {
-              credentials: 'include',
-            });
-            return res.json();
-          },
-          { url: `${API_BASE_URL}/api/auth/status` }
-        );
+        const statusRes = await page.request.get(`${API_BASE_URL}/api/auth/status`, {
+          timeout: 5000,
+        });
+        const statusResponse = (await statusRes.json().catch(() => null)) as {
+          authenticated?: boolean;
+        } | null;
 
-        if (statusResponse.authenticated === true) {
+        if (statusResponse?.authenticated === true) {
           return true;
         }
         attempts++;
         // Use a very short wait between polling attempts (this is acceptable for polling)
-        await page.waitForFunction(() => true, { timeout: 50 });
+        await page.waitForTimeout(50);
       }
 
       return false;

apps/ui/tests/utils/core/interactions.ts

@@ -72,15 +72,21 @@ export async function handleLoginScreenIfPresent(page: Page): Promise<boolean> {
     '[data-testid="welcome-view"], [data-testid="board-view"], [data-testid="context-view"], [data-testid="agent-view"]'
   );
 
-  // Race between login screen and actual content
+  const maxWaitMs = 15000;
+
+  // Race between login screen, a delayed redirect to /login, and actual content
   const loginVisible = await Promise.race([
+    page
+      .waitForURL((url) => url.pathname.includes('/login'), { timeout: maxWaitMs })
+      .then(() => true)
+      .catch(() => false),
     loginInput
-      .waitFor({ state: 'visible', timeout: 5000 })
+      .waitFor({ state: 'visible', timeout: maxWaitMs })
       .then(() => true)
       .catch(() => false),
     appContent
       .first()
-      .waitFor({ state: 'visible', timeout: 5000 })
+      .waitFor({ state: 'visible', timeout: maxWaitMs })
       .then(() => false)
       .catch(() => false),
   ]);
@@ -101,8 +107,8 @@ export async function handleLoginScreenIfPresent(page: Page): Promise<boolean> {
 
     // Wait for navigation away from login - either to content or URL change
     await Promise.race([
-      page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 10000 }),
-      appContent.first().waitFor({ state: 'visible', timeout: 10000 }),
+      page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 15000 }),
+      appContent.first().waitFor({ state: 'visible', timeout: 15000 }),
     ]).catch(() => {});
 
     // Wait for page to load

apps/ui/tests/utils/navigation/views.ts

@@ -1,5 +1,6 @@
 import { Page } from '@playwright/test';
 import { clickElement } from '../core/interactions';
+import { handleLoginScreenIfPresent } from '../core/interactions';
 import { waitForElement } from '../core/waiting';
 import { authenticateForTests } from '../api/client';
 
@@ -15,22 +16,8 @@ export async function navigateToBoard(page: Page): Promise<void> {
   await page.goto('/board');
   await page.waitForLoadState('load');
 
-  // Check if we're on the login screen and handle it
-  const loginInput = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreen = await loginInput.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreen) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInput.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/board', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);
 
   // Wait for the board view to be visible
   await waitForElement(page, 'board-view', { timeout: 10000 });
@@ -48,22 +35,8 @@ export async function navigateToContext(page: Page): Promise<void> {
   await page.goto('/context');
   await page.waitForLoadState('load');
 
-  // Check if we're on the login screen and handle it
-  const loginInputCtx = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenCtx = await loginInputCtx.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreenCtx) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputCtx.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/context', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);
 
   // Wait for loading to complete (if present)
   const loadingElement = page.locator('[data-testid="context-view-loading"]');
@@ -127,22 +100,8 @@ export async function navigateToAgent(page: Page): Promise<void> {
   await page.goto('/agent');
   await page.waitForLoadState('load');
 
-  // Check if we're on the login screen and handle it
-  const loginInputAgent = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenAgent = await loginInputAgent.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreenAgent) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputAgent.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/agent', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);
 
   // Wait for the agent view to be visible
   await waitForElement(page, 'agent-view', { timeout: 10000 });
@@ -187,24 +146,8 @@ export async function navigateToWelcome(page: Page): Promise<void> {
   await page.goto('/');
   await page.waitForLoadState('load');
 
-  // Check if we're on the login screen and handle it
-  const loginInputWelcome = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenWelcome = await loginInputWelcome
-    .isVisible({ timeout: 2000 })
-    .catch(() => false);
-  if (isLoginScreenWelcome) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputWelcome.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);
 
   await waitForElement(page, 'welcome-view', { timeout: 10000 });
 }

apps/ui/tests/utils/project/setup.ts

@@ -6,7 +6,7 @@ import { Page } from '@playwright/test';
  */
 const STORE_VERSIONS = {
   APP_STORE: 2, // Must match app-store.ts persist version
-  SETUP_STORE: 0, // setup-store.ts doesn't specify a version, so zustand defaults to 0
+  SETUP_STORE: 1, // Must match setup-store.ts persist version
 } as const;
 
 /**
@@ -56,6 +56,7 @@ export async function setupWelcomeView(
           currentView: 'welcome',
           theme: 'dark',
           sidebarOpen: true,
+          skipSandboxWarning: true,
           apiKeys: { anthropic: '', google: '' },
           chatSessions: [],
           chatHistoryOpen: false,
@@ -135,6 +136,7 @@ export async function setupRealProject(
           currentView: currentProject ? 'board' : 'welcome',
           theme: 'dark',
           sidebarOpen: true,
+          skipSandboxWarning: true,
           apiKeys: { anthropic: '', google: '' },
           chatSessions: [],
           chatHistoryOpen: false,