chore: update CI configuration and enhance test stability

- Added deterministic API key and environment variables in e2e-tests.yml to ensure consistent test behavior. - Refactored CodexProvider tests to improve type safety and mock handling, ensuring reliable test execution. - Updated provider-factory tests to mock installation detection for CodexProvider, enhancing test isolation. - Adjusted Playwright configuration to conditionally use external backend, improving flexibility in test environments. - Enhanced kill-test-servers script to handle external server scenarios, ensuring proper cleanup of test processes. These changes improve the reliability and maintainability of the testing framework, leading to a more stable development experience.
2026-02-01 08:13:37 +00:00 · 2026-01-07 00:27:38 +05:30
parent 96f154d440
commit 251f0fd88e
12 changed files with 194 additions and 160 deletions
--- a/apps/ui/playwright.config.ts
+++ b/apps/ui/playwright.config.ts
@@ -3,6 +3,7 @@ import { defineConfig, devices } from '@playwright/test';
 const port = process.env.TEST_PORT || 3007;
 const serverPort = process.env.TEST_SERVER_PORT || 3008;
 const reuseServer = process.env.TEST_REUSE_SERVER === 'true';
+const useExternalBackend = !!process.env.VITE_SERVER_URL;
 // Always use mock agent for tests (disables rate limiting, uses mock Claude responses)
 const mockAgent = true;

@@ -33,31 +34,36 @@ export default defineConfig({
        webServer: [
          // Backend server - runs with mock agent enabled in CI
          // Uses dev:test (no file watching) to avoid port conflicts from server restarts
-          {
-            command: `cd ../server && npm run dev:test`,
-            url: `http://localhost:${serverPort}/api/health`,
-            // Don't reuse existing server to ensure we use the test API key
-            reuseExistingServer: false,
-            timeout: 60000,
-            env: {
-              ...process.env,
-              PORT: String(serverPort),
-              // Enable mock agent in CI to avoid real API calls
-              AUTOMAKER_MOCK_AGENT: mockAgent ? 'true' : 'false',
-              // Set a test API key for web mode authentication
-              AUTOMAKER_API_KEY: process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests',
-              // Hide the API key banner to reduce log noise
-              AUTOMAKER_HIDE_API_KEY: 'true',
-              // No ALLOWED_ROOT_DIRECTORY restriction - allow all paths for testing
-              // Simulate containerized environment to skip sandbox confirmation dialogs
-              IS_CONTAINERIZED: 'true',
-            },
-          },
+          ...(useExternalBackend
+            ? []
+            : [
+                {
+                  command: `cd ../server && npm run dev:test`,
+                  url: `http://localhost:${serverPort}/api/health`,
+                  // Don't reuse existing server to ensure we use the test API key
+                  reuseExistingServer: false,
+                  timeout: 60000,
+                  env: {
+                    ...process.env,
+                    PORT: String(serverPort),
+                    // Enable mock agent in CI to avoid real API calls
+                    AUTOMAKER_MOCK_AGENT: mockAgent ? 'true' : 'false',
+                    // Set a test API key for web mode authentication
+                    AUTOMAKER_API_KEY:
+                      process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests',
+                    // Hide the API key banner to reduce log noise
+                    AUTOMAKER_HIDE_API_KEY: 'true',
+                    // No ALLOWED_ROOT_DIRECTORY restriction - allow all paths for testing
+                    // Simulate containerized environment to skip sandbox confirmation dialogs
+                    IS_CONTAINERIZED: 'true',
+                  },
+                },
+              ]),
          // Frontend Vite dev server
          {
            command: `npm run dev`,
            url: `http://localhost:${port}`,
-            reuseExistingServer: true,
+            reuseExistingServer: false,
            timeout: 120000,
            env: {
              ...process.env,
--- a/apps/ui/scripts/kill-test-servers.mjs
+++ b/apps/ui/scripts/kill-test-servers.mjs
@@ -10,24 +10,42 @@ const execAsync = promisify(exec);

 const SERVER_PORT = process.env.TEST_SERVER_PORT || 3008;
 const UI_PORT = process.env.TEST_PORT || 3007;
+const USE_EXTERNAL_SERVER = !!process.env.VITE_SERVER_URL;

 async function killProcessOnPort(port) {
  try {
-    const { stdout } = await execAsync(`lsof -ti:${port}`);
-    const pids = stdout.trim().split('\n').filter(Boolean);
+    const hasLsof = await execAsync('command -v lsof').then(
+      () => true,
+      () => false
+    );

-    if (pids.length > 0) {
-      console.log(`[KillTestServers] Found process(es) on port ${port}: ${pids.join(', ')}`);
-      for (const pid of pids) {
-        try {
-          await execAsync(`kill -9 ${pid}`);
-          console.log(`[KillTestServers] Killed process ${pid}`);
-        } catch (error) {
-          // Process might have already exited
+    if (hasLsof) {
+      const { stdout } = await execAsync(`lsof -ti:${port}`);
+      const pids = stdout.trim().split('\n').filter(Boolean);
+
+      if (pids.length > 0) {
+        console.log(`[KillTestServers] Found process(es) on port ${port}: ${pids.join(', ')}`);
+        for (const pid of pids) {
+          try {
+            await execAsync(`kill -9 ${pid}`);
+            console.log(`[KillTestServers] Killed process ${pid}`);
+          } catch (error) {
+            // Process might have already exited
+          }
        }
+        await new Promise((resolve) => setTimeout(resolve, 500));
      }
-      // Wait a moment for the port to be released
+      return;
+    }
+
+    const hasFuser = await execAsync('command -v fuser').then(
+      () => true,
+      () => false
+    );
+    if (hasFuser) {
+      await execAsync(`fuser -k -9 ${port}/tcp`).catch(() => undefined);
      await new Promise((resolve) => setTimeout(resolve, 500));
+      return;
    }
  } catch (error) {
    // No process on port, which is fine
@@ -36,7 +54,9 @@ async function killProcessOnPort(port) {

 async function main() {
  console.log('[KillTestServers] Checking for existing test servers...');
-  await killProcessOnPort(Number(SERVER_PORT));
+  if (!USE_EXTERNAL_SERVER) {
+    await killProcessOnPort(Number(SERVER_PORT));
+  }
  await killProcessOnPort(Number(UI_PORT));
  console.log('[KillTestServers] Done');
 }
--- a/apps/ui/src/lib/http-api-client.ts
+++ b/apps/ui/src/lib/http-api-client.ts
@@ -349,6 +349,7 @@ export const verifySession = async (): Promise<boolean> => {
    const response = await fetch(`${getServerUrl()}/api/settings/status`, {
      headers,
      credentials: 'include',
+      signal: AbortSignal.timeout(5000),
    });

    // Check for authentication errors
@@ -390,6 +391,7 @@ export const checkSandboxEnvironment = async (): Promise<{
  try {
    const response = await fetch(`${getServerUrl()}/api/health/environment`, {
      method: 'GET',
+      signal: AbortSignal.timeout(5000),
    });

    if (!response.ok) {
--- a/apps/ui/tests/utils/api/client.ts
+++ b/apps/ui/tests/utils/api/client.ts
@@ -282,28 +282,40 @@ export async function apiListBranches(
 */
 export async function authenticateWithApiKey(page: Page, apiKey: string): Promise<boolean> {
  try {
+    // Ensure the backend is up before attempting login (especially in local runs where
+    // the backend may be started separately from Playwright).
+    const start = Date.now();
+    while (Date.now() - start < 15000) {
+      try {
+        const health = await page.request.get(`${API_BASE_URL}/api/health`, {
+          timeout: 3000,
+        });
+        if (health.ok()) break;
+      } catch {
+        // Retry
+      }
+      await page.waitForTimeout(250);
+    }
+
    // Ensure we're on a page (needed for cookies to work)
    const currentUrl = page.url();
    if (!currentUrl || currentUrl === 'about:blank') {
      await page.goto('http://localhost:3007', { waitUntil: 'domcontentloaded' });
    }

-    // Use browser context fetch to ensure cookies are set in the browser
-    const response = await page.evaluate(
-      async ({ url, apiKey }) => {
-        const res = await fetch(url, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          credentials: 'include',
-          body: JSON.stringify({ apiKey }),
-        });
-        const data = await res.json();
-        return { success: data.success, token: data.token };
-      },
-      { url: `${API_BASE_URL}/api/auth/login`, apiKey }
-    );
+    // Use Playwright request API (tied to this browser context) to avoid flakiness
+    // with cross-origin fetch inside page.evaluate.
+    const loginResponse = await page.request.post(`${API_BASE_URL}/api/auth/login`, {
+      data: { apiKey },
+      headers: { 'Content-Type': 'application/json' },
+      timeout: 15000,
+    });
+    const response = (await loginResponse.json().catch(() => null)) as {
+      success?: boolean;
+      token?: string;
+    } | null;

-    if (response.success && response.token) {
+    if (response?.success && response.token) {
      // Manually set the cookie in the browser context
      // The server sets a cookie named 'automaker_session' (see SESSION_COOKIE_NAME in auth.ts)
      await page.context().addCookies([
@@ -322,22 +334,19 @@ export async function authenticateWithApiKey(page: Page, apiKey: string): Promis
      let attempts = 0;
      const maxAttempts = 10;
      while (attempts < maxAttempts) {
-        const statusResponse = await page.evaluate(
-          async ({ url }) => {
-            const res = await fetch(url, {
-              credentials: 'include',
-            });
-            return res.json();
-          },
-          { url: `${API_BASE_URL}/api/auth/status` }
-        );
+        const statusRes = await page.request.get(`${API_BASE_URL}/api/auth/status`, {
+          timeout: 5000,
+        });
+        const statusResponse = (await statusRes.json().catch(() => null)) as {
+          authenticated?: boolean;
+        } | null;

-        if (statusResponse.authenticated === true) {
+        if (statusResponse?.authenticated === true) {
          return true;
        }
        attempts++;
        // Use a very short wait between polling attempts (this is acceptable for polling)
-        await page.waitForFunction(() => true, { timeout: 50 });
+        await page.waitForTimeout(50);
      }

      return false;
--- a/apps/ui/tests/utils/core/interactions.ts
+++ b/apps/ui/tests/utils/core/interactions.ts
@@ -72,15 +72,21 @@ export async function handleLoginScreenIfPresent(page: Page): Promise<boolean> {
    '[data-testid="welcome-view"], [data-testid="board-view"], [data-testid="context-view"], [data-testid="agent-view"]'
  );

-  // Race between login screen and actual content
+  const maxWaitMs = 15000;
+
+  // Race between login screen, a delayed redirect to /login, and actual content
  const loginVisible = await Promise.race([
+    page
+      .waitForURL((url) => url.pathname.includes('/login'), { timeout: maxWaitMs })
+      .then(() => true)
+      .catch(() => false),
    loginInput
-      .waitFor({ state: 'visible', timeout: 5000 })
+      .waitFor({ state: 'visible', timeout: maxWaitMs })
      .then(() => true)
      .catch(() => false),
    appContent
      .first()
-      .waitFor({ state: 'visible', timeout: 5000 })
+      .waitFor({ state: 'visible', timeout: maxWaitMs })
      .then(() => false)
      .catch(() => false),
  ]);
@@ -101,8 +107,8 @@ export async function handleLoginScreenIfPresent(page: Page): Promise<boolean> {

    // Wait for navigation away from login - either to content or URL change
    await Promise.race([
-      page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 10000 }),
-      appContent.first().waitFor({ state: 'visible', timeout: 10000 }),
+      page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 15000 }),
+      appContent.first().waitFor({ state: 'visible', timeout: 15000 }),
    ]).catch(() => {});

    // Wait for page to load
--- a/apps/ui/tests/utils/navigation/views.ts
+++ b/apps/ui/tests/utils/navigation/views.ts
@@ -1,5 +1,6 @@
 import { Page } from '@playwright/test';
 import { clickElement } from '../core/interactions';
+import { handleLoginScreenIfPresent } from '../core/interactions';
 import { waitForElement } from '../core/waiting';
 import { authenticateForTests } from '../api/client';

@@ -15,22 +16,8 @@ export async function navigateToBoard(page: Page): Promise<void> {
  await page.goto('/board');
  await page.waitForLoadState('load');

-  // Check if we're on the login screen and handle it
-  const loginInput = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreen = await loginInput.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreen) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInput.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/board', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);

  // Wait for the board view to be visible
  await waitForElement(page, 'board-view', { timeout: 10000 });
@@ -48,22 +35,8 @@ export async function navigateToContext(page: Page): Promise<void> {
  await page.goto('/context');
  await page.waitForLoadState('load');

-  // Check if we're on the login screen and handle it
-  const loginInputCtx = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenCtx = await loginInputCtx.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreenCtx) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputCtx.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/context', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);

  // Wait for loading to complete (if present)
  const loadingElement = page.locator('[data-testid="context-view-loading"]');
@@ -127,22 +100,8 @@ export async function navigateToAgent(page: Page): Promise<void> {
  await page.goto('/agent');
  await page.waitForLoadState('load');

-  // Check if we're on the login screen and handle it
-  const loginInputAgent = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenAgent = await loginInputAgent.isVisible({ timeout: 2000 }).catch(() => false);
-  if (isLoginScreenAgent) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputAgent.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/agent', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);

  // Wait for the agent view to be visible
  await waitForElement(page, 'agent-view', { timeout: 10000 });
@@ -187,24 +146,8 @@ export async function navigateToWelcome(page: Page): Promise<void> {
  await page.goto('/');
  await page.waitForLoadState('load');

-  // Check if we're on the login screen and handle it
-  const loginInputWelcome = page
-    .locator('[data-testid="login-api-key-input"], input[type="password"][placeholder*="API key"]')
-    .first();
-  const isLoginScreenWelcome = await loginInputWelcome
-    .isVisible({ timeout: 2000 })
-    .catch(() => false);
-  if (isLoginScreenWelcome) {
-    const apiKey = process.env.AUTOMAKER_API_KEY || 'test-api-key-for-e2e-tests';
-    await loginInputWelcome.fill(apiKey);
-    await page.waitForTimeout(100);
-    await page
-      .locator('[data-testid="login-submit-button"], button:has-text("Login")')
-      .first()
-      .click();
-    await page.waitForURL('**/', { timeout: 5000 });
-    await page.waitForLoadState('load');
-  }
+  // Handle login redirect if needed
+  await handleLoginScreenIfPresent(page);

  await waitForElement(page, 'welcome-view', { timeout: 10000 });
 }
--- a/apps/ui/tests/utils/project/setup.ts
+++ b/apps/ui/tests/utils/project/setup.ts
@@ -6,7 +6,7 @@ import { Page } from '@playwright/test';
 */
 const STORE_VERSIONS = {
  APP_STORE: 2, // Must match app-store.ts persist version
-  SETUP_STORE: 0, // setup-store.ts doesn't specify a version, so zustand defaults to 0
+  SETUP_STORE: 1, // Must match setup-store.ts persist version
 } as const;

 /**
@@ -56,6 +56,7 @@ export async function setupWelcomeView(
          currentView: 'welcome',
          theme: 'dark',
          sidebarOpen: true,
+          skipSandboxWarning: true,
          apiKeys: { anthropic: '', google: '' },
          chatSessions: [],
          chatHistoryOpen: false,
@@ -135,6 +136,7 @@ export async function setupRealProject(
          currentView: currentProject ? 'board' : 'welcome',
          theme: 'dark',
          sidebarOpen: true,
+          skipSandboxWarning: true,
          apiKeys: { anthropic: '', google: '' },
          chatSessions: [],
          chatHistoryOpen: false,