Fix agent output validation to prevent false verified status (#807)

* Changes from fix/cursor-fix

* feat: Enhance provider error messages with diagnostic context, address test failure, fix port change, move playwright tests to different port

* Update apps/ui/src/components/views/board-view/dialogs/add-feature-dialog.tsx

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* ci: Update test server port from 3008 to 3108 and add environment configuration

* fix: Correct typo in health endpoint URL and standardize port env vars

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

apps/ui/tests/utils/api/client.ts

@@ -4,7 +4,7 @@
  */
 
 import { Page, APIResponse } from '@playwright/test';
-import { API_BASE_URL, API_ENDPOINTS } from '../core/constants';
+import { API_BASE_URL, API_ENDPOINTS, WEB_BASE_URL } from '../core/constants';
 
 // ============================================================================
 // Types
@@ -300,7 +300,7 @@ export async function authenticateWithApiKey(page: Page, apiKey: string): Promis
     // Ensure we're on a page (needed for cookies to work)
     const currentUrl = page.url();
     if (!currentUrl || currentUrl === 'about:blank') {
-      await page.goto('http://localhost:3007', { waitUntil: 'domcontentloaded' });
+      await page.goto(WEB_BASE_URL, { waitUntil: 'domcontentloaded' });
     }
 
     // Use Playwright request API (tied to this browser context) to avoid flakiness

apps/ui/tests/utils/core/constants.ts

@@ -9,8 +9,19 @@
 
 /**
  * Base URL for the API server
+ * Uses TEST_SERVER_PORT env var (default 3108) for test runs
  */
-export const API_BASE_URL = 'http://localhost:3008';
+export const API_BASE_URL = process.env.TEST_SERVER_PORT
+  ? `http://localhost:${process.env.TEST_SERVER_PORT}`
+  : 'http://localhost:3108';
+
+/**
+ * Base URL for the frontend web server
+ * Uses TEST_PORT env var (default 3107) for test runs
+ */
+export const WEB_BASE_URL = process.env.TEST_PORT
+  ? `http://localhost:${process.env.TEST_PORT}`
+  : 'http://localhost:3107';
 
 /**
  * API endpoints for worktree operations

apps/ui/tests/utils/core/waiting.ts

@@ -1,5 +1,12 @@
 import { Page, Locator } from '@playwright/test';
 
+/**
+ * Default timeout for element waiting operations in E2E tests.
+ * Increased from 5000ms to 10000ms to accommodate CI environments
+ * where dialog rendering may take longer due to React Query data fetching.
+ */
+export const DEFAULT_ELEMENT_TIMEOUT_MS = 10000;
+
 /**
  * Wait for the page to load
  * Uses 'load' state instead of 'networkidle' because the app has persistent
@@ -20,7 +27,7 @@ export async function waitForElement(
 ): Promise<Locator> {
   const element = page.locator(`[data-testid="${testId}"]`);
   await element.waitFor({
-    timeout: options?.timeout ?? 5000,
+    timeout: options?.timeout ?? DEFAULT_ELEMENT_TIMEOUT_MS,
     state: options?.state ?? 'visible',
   });
   return element;
@@ -36,7 +43,7 @@ export async function waitForElementHidden(
 ): Promise<void> {
   const element = page.locator(`[data-testid="${testId}"]`);
   await element.waitFor({
-    timeout: options?.timeout ?? 5000,
+    timeout: options?.timeout ?? DEFAULT_ELEMENT_TIMEOUT_MS,
     state: 'hidden',
   });
 }

apps/ui/tests/utils/views/board.ts

@@ -1,4 +1,5 @@
 import { Page, Locator } from '@playwright/test';
+import { DEFAULT_ELEMENT_TIMEOUT_MS } from '../core/waiting';
 
 /**
  * Get a kanban card by feature ID
@@ -96,9 +97,16 @@ export async function getDragHandleForFeature(page: Page, featureId: string): Pr
  * Click the add feature button
  */
 export async function clickAddFeature(page: Page): Promise<void> {
-  await page.click('[data-testid="add-feature-button"]');
+  // There may be multiple add-feature buttons on the page (header, empty state).
+  // Use .first() to click the first visible one.
+  const addButton = page.locator('[data-testid="add-feature-button"]').first();
+  await addButton.waitFor({ state: 'visible', timeout: DEFAULT_ELEMENT_TIMEOUT_MS });
+  await addButton.click({ timeout: 5000 });
+
+  // Wait for dialog to be visible
   await page.waitForSelector('[data-testid="add-feature-dialog"]', {
-    timeout: 5000,
+    state: 'visible',
+    timeout: DEFAULT_ELEMENT_TIMEOUT_MS,
   });
 }
 
@@ -159,7 +167,7 @@ export async function confirmAddFeature(page: Page): Promise<void> {
   await page.click('[data-testid="confirm-add-feature"]');
   // Wait for dialog to close
   await page.waitForFunction(() => !document.querySelector('[data-testid="add-feature-dialog"]'), {
-    timeout: 5000,
+    timeout: DEFAULT_ELEMENT_TIMEOUT_MS,
   });
 }