feat: add end-to-end testing guide and project creation tests

- Introduced a comprehensive E2E Testing Guide outlining best practices for Playwright tests, including principles for test isolation, element selection, and setup utilities.
- Added new test files for project creation and opening existing projects, ensuring functionality for creating blank projects and projects from GitHub templates.
- Implemented utility functions for setting up test states and managing localStorage, enhancing maintainability and reducing boilerplate in tests.
This commit is contained in:
Test User
2025-12-22 12:49:48 -05:00
parent 3a43033fa6
commit 0c508ce130
4 changed files with 781 additions and 3 deletions

View File

@@ -0,0 +1,306 @@
# E2E Testing Guide
Best practices and patterns for writing reliable, non-flaky Playwright e2e tests in this codebase.
## Core Principles
1. **No arbitrary timeouts** - Never use `page.waitForTimeout()`. Always wait for specific conditions.
2. **Use data-testid attributes** - Prefer `[data-testid="..."]` selectors over CSS classes or text content.
3. **Clean up after tests** - Use unique temp directories and clean them up in `afterAll`.
4. **Test isolation** - Each test should be independent and not rely on state from other tests.
## Setting Up Test State
### Use Setup Utilities (Recommended)
Use the provided utility functions to set up localStorage state. These utilities hide the internal store structure and version details, making tests more maintainable.
```typescript
import { setupWelcomeView, setupRealProject } from './utils';
// Show welcome view with workspace directory configured
await setupWelcomeView(page, { workspaceDir: TEST_TEMP_DIR });
// Show welcome view with recent projects
await setupWelcomeView(page, {
workspaceDir: TEST_TEMP_DIR,
recentProjects: [
{
id: 'project-123',
name: 'My Project',
path: '/path/to/project',
lastOpened: new Date().toISOString(),
},
],
});
// Set up a real project on the filesystem
await setupRealProject(page, projectPath, projectName, {
setAsCurrent: true, // Opens board view (default)
});
```
### Why Use Utilities Instead of Raw localStorage
1. **Version management** - Store versions are centralized in one place
2. **Less brittle** - If store structure changes, update one file instead of every test
3. **Cleaner tests** - Focus on test logic, not setup boilerplate
4. **Type safety** - Utilities provide typed interfaces for test data
### Manual LocalStorage Setup (Advanced)
If you need custom setup not covered by utilities, use `page.addInitScript()`.
Store versions are defined in `tests/utils/project/setup.ts`:
- `APP_STORE`: version 2 (matches `app-store.ts`)
- `SETUP_STORE`: version 0 (matches `setup-store.ts` default)
### Temp Directory Management
Create unique temp directories for test isolation:
```typescript
import { createTempDirPath, cleanupTempDir } from './utils';
const TEST_TEMP_DIR = createTempDirPath('my-test-name');
test.describe('My Tests', () => {
test.beforeAll(async () => {
if (!fs.existsSync(TEST_TEMP_DIR)) {
fs.mkdirSync(TEST_TEMP_DIR, { recursive: true });
}
});
test.afterAll(async () => {
cleanupTempDir(TEST_TEMP_DIR);
});
});
```
## Waiting for Elements
### Prefer `toBeVisible()` over `waitForSelector()`
```typescript
// Good - uses Playwright's auto-waiting with expect
await expect(page.locator('[data-testid="welcome-view"]')).toBeVisible({ timeout: 10000 });
// Avoid - manual waiting
await page.waitForSelector('[data-testid="welcome-view"]');
```
### Wait for network idle after navigation
```typescript
await page.goto('/');
await page.waitForLoadState('networkidle');
```
### Use appropriate timeouts
- Quick UI updates: 5000ms (default)
- Page loads/navigation: 10000ms
- Async operations (API calls, file system): 15000ms
```typescript
// Fast UI element
await expect(button).toBeVisible();
// Page load
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 10000 });
// Async operation completion
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 15000 });
```
## Element Selection
### Use data-testid attributes
```typescript
// Good - stable selector
const button = page.locator('[data-testid="create-new-project"]');
// Avoid - brittle selectors
const button = page.locator('.btn-primary');
const button = page.getByText('Create');
```
### Scope selectors when needed
When text appears in multiple places, scope to a parent:
```typescript
// Bad - might match multiple elements
await expect(page.getByText(projectName)).toBeVisible();
// Good - scoped to specific container
await expect(page.locator('[data-testid="project-selector"]').getByText(projectName)).toBeVisible();
```
### Handle strict mode violations
If a selector matches multiple elements:
```typescript
// Use .first() if you need the first match
await page.locator('[data-testid="item"]').first().click();
// Or scope to a unique parent
await page.locator('[data-testid="sidebar"]').locator('[data-testid="item"]').click();
```
## Clicking Elements
### Always verify visibility before clicking
```typescript
const button = page.locator('[data-testid="submit"]');
await expect(button).toBeVisible();
await button.click();
```
### Handle dialogs that may close quickly
Some dialogs may appear briefly or auto-close. Don't rely on clicking them:
```typescript
// Instead of trying to close a dialog that might disappear:
// await expect(dialog).toBeVisible();
// await closeButton.click(); // May fail if dialog closes first
// Just verify the end state:
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 15000 });
```
## Filesystem Verification
Verify files were created after async operations:
```typescript
// Wait for UI to confirm operation completed first
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 15000 });
// Then verify filesystem
const projectPath = path.join(TEST_TEMP_DIR, projectName);
expect(fs.existsSync(projectPath)).toBe(true);
const appSpecPath = path.join(projectPath, '.automaker', 'app_spec.txt');
expect(fs.existsSync(appSpecPath)).toBe(true);
const content = fs.readFileSync(appSpecPath, 'utf-8');
expect(content).toContain(projectName);
```
## Test Structure
### Use descriptive test names
```typescript
test('should create a new blank project from welcome view', async ({ page }) => {
// ...
});
```
### Group related tests with describe blocks
```typescript
test.describe('Project Creation', () => {
test('should create a new blank project from welcome view', ...);
test('should create a project from template', ...);
});
```
### Use serial mode when tests depend on each other
```typescript
test.describe.configure({ mode: 'serial' });
```
## Common Patterns
### Waiting for either of two outcomes
When multiple outcomes are possible (e.g., dialog or direct navigation):
```typescript
// Wait for either the dialog or the board view
await Promise.race([
initDialog.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {}),
boardView.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {}),
]);
// Then handle whichever appeared
if (await initDialog.isVisible()) {
await closeButton.click();
}
await expect(boardView).toBeVisible();
```
### Generating unique test data
```typescript
const projectName = `test-project-${Date.now()}`;
```
## Running Tests
```bash
# Run all tests
npm run test
# Run specific test file
npm run test -- project-creation.spec.ts
# Run with headed browser (see what's happening)
npm run test:headed -- project-creation.spec.ts
# Run multiple times to check for flakiness
npm run test -- project-creation.spec.ts --repeat-each=5
```
## Debugging Failed Tests
1. Check the screenshot in `test-results/`
2. Read the error context markdown file in `test-results/`
3. Run with `--headed` to watch the test
4. Add `await page.pause()` to pause execution at a specific point
## Available Test Utilities
Import from `./utils`:
### State Setup Utilities
- `setupWelcomeView(page, options?)` - Set up empty state showing welcome view
- `options.workspaceDir` - Pre-configure workspace directory
- `options.recentProjects` - Add projects to recent list (not current)
- `setupRealProject(page, path, name, options?)` - Set up state with a real filesystem project
- `options.setAsCurrent` - Open board view (default: true)
- `options.additionalProjects` - Add more projects to list
- `setupMockProject(page)` - Set up mock project for unit-style tests
- `setupComplete(page)` - Mark setup wizard as complete
### Filesystem Utilities
- `createTempDirPath(prefix)` - Create unique temp directory path
- `cleanupTempDir(path)` - Remove temp directory
- `createTestGitRepo(tempDir)` - Create a git repo for testing
### Waiting Utilities
- `waitForNetworkIdle(page)` - Wait for network to be idle
- `waitForElement(page, testId)` - Wait for element by test ID
### Async File Verification
Use `expect().toPass()` for polling filesystem operations:
```typescript
await expect(async () => {
expect(fs.existsSync(filePath)).toBe(true);
}).toPass({ timeout: 10000 });
```
See `tests/utils/index.ts` for the full list of available utilities.

View File

@@ -0,0 +1,125 @@
/**
* Open Project End-to-End Test
*
* Tests opening an existing project directory from the welcome view.
* This verifies that:
* 1. An existing directory can be opened as a project
* 2. The .automaker directory is initialized if it doesn't exist
* 3. The project is loaded and shown in the board view
*/
import { test, expect } from '@playwright/test';
import * as fs from 'fs';
import * as path from 'path';
import { createTempDirPath, cleanupTempDir, setupWelcomeView } from './utils';
// Create unique temp dir for this test run
const TEST_TEMP_DIR = createTempDirPath('open-project-test');
test.describe('Open Project', () => {
test.beforeAll(async () => {
// Create test temp directory
if (!fs.existsSync(TEST_TEMP_DIR)) {
fs.mkdirSync(TEST_TEMP_DIR, { recursive: true });
}
});
test.afterAll(async () => {
// Cleanup temp directory
cleanupTempDir(TEST_TEMP_DIR);
});
test('should open an existing project directory from recent projects', async ({ page }) => {
const projectName = `existing-project-${Date.now()}`;
const projectPath = path.join(TEST_TEMP_DIR, projectName);
const projectId = `project-${Date.now()}`;
// Create the project directory with some files to simulate an existing codebase
fs.mkdirSync(projectPath, { recursive: true });
// Create a package.json to simulate a real project
fs.writeFileSync(
path.join(projectPath, 'package.json'),
JSON.stringify(
{
name: projectName,
version: '1.0.0',
description: 'A test project for e2e testing',
},
null,
2
)
);
// Create a README.md
fs.writeFileSync(path.join(projectPath, 'README.md'), `# ${projectName}\n\nA test project.`);
// Create a src directory with an index.ts file
fs.mkdirSync(path.join(projectPath, 'src'), { recursive: true });
fs.writeFileSync(
path.join(projectPath, 'src', 'index.ts'),
'export const hello = () => console.log("Hello World");'
);
// Set up welcome view with the project in recent projects (but NOT as current project)
await setupWelcomeView(page, {
recentProjects: [
{
id: projectId,
name: projectName,
path: projectPath,
lastOpened: new Date(Date.now() - 86400000).toISOString(), // 1 day ago
},
],
});
// Navigate to the app
await page.goto('/');
await page.waitForLoadState('networkidle');
// Wait for welcome view to be visible
await expect(page.locator('[data-testid="welcome-view"]')).toBeVisible({ timeout: 10000 });
// Verify we see the "Recent Projects" section
await expect(page.getByText('Recent Projects')).toBeVisible({ timeout: 5000 });
// Click on the recent project to open it
const recentProjectCard = page.locator(`[data-testid="recent-project-${projectId}"]`);
await expect(recentProjectCard).toBeVisible();
await recentProjectCard.click();
// Wait for the board view to appear (project was opened)
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 15000 });
// Verify the project name appears in the project selector (sidebar)
await expect(
page.locator('[data-testid="project-selector"]').getByText(projectName)
).toBeVisible({ timeout: 5000 });
// Verify .automaker directory was created (initialized for the first time)
// Use polling since file creation may be async
const automakerDir = path.join(projectPath, '.automaker');
await expect(async () => {
expect(fs.existsSync(automakerDir)).toBe(true);
}).toPass({ timeout: 10000 });
// Verify the required structure was created by initializeProject:
// - .automaker/categories.json
// - .automaker/features directory
// - .automaker/context directory
// Note: app_spec.txt is NOT created automatically for existing projects
const categoriesPath = path.join(automakerDir, 'categories.json');
await expect(async () => {
expect(fs.existsSync(categoriesPath)).toBe(true);
}).toPass({ timeout: 10000 });
// Verify subdirectories were created
expect(fs.existsSync(path.join(automakerDir, 'features'))).toBe(true);
expect(fs.existsSync(path.join(automakerDir, 'context'))).toBe(true);
// Verify the original project files still exist (weren't modified)
expect(fs.existsSync(path.join(projectPath, 'package.json'))).toBe(true);
expect(fs.existsSync(path.join(projectPath, 'README.md'))).toBe(true);
expect(fs.existsSync(path.join(projectPath, 'src', 'index.ts'))).toBe(true);
});
});

View File

@@ -0,0 +1,188 @@
/**
* Project Creation End-to-End Tests
*
* Tests the project creation flows:
* 1. Creating a new blank project from the welcome view
* 2. Creating a new project from a GitHub template
*/
import { test, expect } from '@playwright/test';
import * as fs from 'fs';
import * as path from 'path';
import { createTempDirPath, cleanupTempDir, setupWelcomeView } from './utils';
// Create unique temp dir for this test run
const TEST_TEMP_DIR = createTempDirPath('project-creation-test');
test.describe('Project Creation', () => {
test.beforeAll(async () => {
// Create test temp directory
if (!fs.existsSync(TEST_TEMP_DIR)) {
fs.mkdirSync(TEST_TEMP_DIR, { recursive: true });
}
});
test.afterAll(async () => {
// Cleanup temp directory
cleanupTempDir(TEST_TEMP_DIR);
});
test('should create a new blank project from welcome view', async ({ page }) => {
const projectName = `test-project-${Date.now()}`;
// Set up welcome view with workspace directory pre-configured
await setupWelcomeView(page, { workspaceDir: TEST_TEMP_DIR });
// Navigate to the app
await page.goto('/');
await page.waitForLoadState('networkidle');
// Wait for welcome view to be visible
await expect(page.locator('[data-testid="welcome-view"]')).toBeVisible({ timeout: 10000 });
// Click the "Create New Project" dropdown button
const createButton = page.locator('[data-testid="create-new-project"]');
await expect(createButton).toBeVisible();
await createButton.click();
// Click "Quick Setup" option from the dropdown
const quickSetupOption = page.locator('[data-testid="quick-setup-option"]');
await expect(quickSetupOption).toBeVisible();
await quickSetupOption.click();
// Wait for the new project modal to appear
const modal = page.locator('[data-testid="new-project-modal"]');
await expect(modal).toBeVisible({ timeout: 5000 });
// Enter the project name
const projectNameInput = page.locator('[data-testid="project-name-input"]');
await expect(projectNameInput).toBeVisible();
await projectNameInput.fill(projectName);
// Verify the workspace directory is shown (from our pre-configured localStorage)
// Wait for workspace to be loaded (it shows "Will be created at:" when ready)
await expect(page.getByText('Will be created at:')).toBeVisible({ timeout: 5000 });
// Click the Create Project button
const createProjectButton = page.locator('[data-testid="confirm-create-project"]');
await expect(createProjectButton).toBeVisible();
await createProjectButton.click();
// Wait for project creation to complete
// The app may show an init dialog briefly and then navigate to board view
// We just need to verify we end up on the board view with our project
// Wait for the board view - this confirms the project was created and opened
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 15000 });
// Verify the project name appears in the project selector (sidebar)
await expect(
page.locator('[data-testid="project-selector"]').getByText(projectName)
).toBeVisible({ timeout: 5000 });
// Verify the project was created in the filesystem
const projectPath = path.join(TEST_TEMP_DIR, projectName);
expect(fs.existsSync(projectPath)).toBe(true);
// Verify .automaker directory was created
const automakerDir = path.join(projectPath, '.automaker');
expect(fs.existsSync(automakerDir)).toBe(true);
// Verify app_spec.txt was created
const appSpecPath = path.join(automakerDir, 'app_spec.txt');
expect(fs.existsSync(appSpecPath)).toBe(true);
// Verify the app_spec.txt contains the project name
const appSpecContent = fs.readFileSync(appSpecPath, 'utf-8');
expect(appSpecContent).toContain(projectName);
});
test('should create a new project from GitHub template', async ({ page }) => {
// Increase timeout for this test since git clone takes time
test.setTimeout(60000);
const projectName = `template-project-${Date.now()}`;
// Set up welcome view with workspace directory pre-configured
await setupWelcomeView(page, { workspaceDir: TEST_TEMP_DIR });
// Navigate to the app
await page.goto('/');
await page.waitForLoadState('networkidle');
// Wait for welcome view to be visible
await expect(page.locator('[data-testid="welcome-view"]')).toBeVisible({ timeout: 10000 });
// Click the "Create New Project" dropdown button
const createButton = page.locator('[data-testid="create-new-project"]');
await expect(createButton).toBeVisible();
await createButton.click();
// Click "Quick Setup" option from the dropdown
const quickSetupOption = page.locator('[data-testid="quick-setup-option"]');
await expect(quickSetupOption).toBeVisible();
await quickSetupOption.click();
// Wait for the new project modal to appear
const modal = page.locator('[data-testid="new-project-modal"]');
await expect(modal).toBeVisible({ timeout: 5000 });
// Enter the project name first
const projectNameInput = page.locator('[data-testid="project-name-input"]');
await expect(projectNameInput).toBeVisible();
await projectNameInput.fill(projectName);
// Wait for workspace directory to be loaded
await expect(page.getByText('Will be created at:')).toBeVisible({ timeout: 5000 });
// Click on the "Starter Kit" tab
const starterKitTab = modal.getByText('Starter Kit');
await expect(starterKitTab).toBeVisible();
await starterKitTab.click();
// Select the first template (Automaker Starter Kit)
const firstTemplate = page.locator('[data-testid="template-automaker-starter-kit"]');
await expect(firstTemplate).toBeVisible();
await firstTemplate.click();
// Verify the template is selected (check mark should appear)
await expect(firstTemplate.locator('.lucide-check')).toBeVisible();
// Click the Create Project button
const createProjectButton = page.locator('[data-testid="confirm-create-project"]');
await expect(createProjectButton).toBeVisible();
await createProjectButton.click();
// Wait for git clone to complete and board view to appear
// This takes longer due to the git clone operation
await expect(page.locator('[data-testid="board-view"]')).toBeVisible({ timeout: 45000 });
// Verify the project name appears in the project selector (sidebar)
await expect(
page.locator('[data-testid="project-selector"]').getByText(projectName)
).toBeVisible({ timeout: 5000 });
// Verify the project was cloned in the filesystem
const projectPath = path.join(TEST_TEMP_DIR, projectName);
expect(fs.existsSync(projectPath)).toBe(true);
// Verify .automaker directory was created
const automakerDir = path.join(projectPath, '.automaker');
expect(fs.existsSync(automakerDir)).toBe(true);
// Verify app_spec.txt was created with template info
const appSpecPath = path.join(automakerDir, 'app_spec.txt');
expect(fs.existsSync(appSpecPath)).toBe(true);
const appSpecContent = fs.readFileSync(appSpecPath, 'utf-8');
expect(appSpecContent).toContain(projectName);
expect(appSpecContent).toContain('Automaker Starter Kit');
// Verify the template files were cloned (check for package.json which should exist in the template)
const packageJsonPath = path.join(projectPath, 'package.json');
expect(fs.existsSync(packageJsonPath)).toBe(true);
// Verify it's a git repository (cloned from GitHub)
const gitDir = path.join(projectPath, '.git');
expect(fs.existsSync(gitDir)).toBe(true);
});
});

View File

@@ -1,5 +1,164 @@
import { Page } from '@playwright/test';
/**
* Store version constants - centralized to avoid hardcoding across tests
* These MUST match the versions used in the actual stores
*/
const STORE_VERSIONS = {
APP_STORE: 2, // Must match app-store.ts persist version
SETUP_STORE: 0, // setup-store.ts doesn't specify a version, so zustand defaults to 0
} as const;
/**
* Project interface for test setup
*/
export interface TestProject {
id: string;
name: string;
path: string;
lastOpened?: string;
}
/**
* Options for setting up the welcome view
*/
export interface WelcomeViewSetupOptions {
/** Directory path to pre-configure as the workspace directory */
workspaceDir?: string;
/** Recent projects to show (but not as current project) */
recentProjects?: TestProject[];
}
/**
* Set up localStorage to show the welcome view with no current project
* This is the cleanest way to test project creation flows
*
* @param page - Playwright page
* @param options - Configuration options
*/
export async function setupWelcomeView(
page: Page,
options?: WelcomeViewSetupOptions
): Promise<void> {
await page.addInitScript(
({
opts,
versions,
}: {
opts: WelcomeViewSetupOptions | undefined;
versions: typeof STORE_VERSIONS;
}) => {
// Set up empty app state (no current project) - shows welcome view
const appState = {
state: {
projects: opts?.recentProjects || [],
currentProject: null,
currentView: 'welcome',
theme: 'dark',
sidebarOpen: true,
apiKeys: { anthropic: '', google: '' },
chatSessions: [],
chatHistoryOpen: false,
maxConcurrency: 3,
},
version: versions.APP_STORE,
};
localStorage.setItem('automaker-storage', JSON.stringify(appState));
// Mark setup as complete to skip the setup wizard
const setupState = {
state: {
isFirstRun: false,
setupComplete: true,
skipClaudeSetup: false,
},
version: versions.SETUP_STORE,
};
localStorage.setItem('automaker-setup', JSON.stringify(setupState));
// Set workspace directory if provided
if (opts?.workspaceDir) {
localStorage.setItem('automaker:lastProjectDir', opts.workspaceDir);
}
},
{ opts: options, versions: STORE_VERSIONS }
);
}
/**
* Set up localStorage with a project at a real filesystem path
* Use this when testing with actual files on disk
*
* @param page - Playwright page
* @param projectPath - Absolute path to the project directory
* @param projectName - Display name for the project
* @param options - Additional options
*/
export async function setupRealProject(
page: Page,
projectPath: string,
projectName: string,
options?: {
/** Set as current project (opens board view) or just add to recent projects */
setAsCurrent?: boolean;
/** Additional recent projects to include */
additionalProjects?: TestProject[];
}
): Promise<void> {
await page.addInitScript(
({
path,
name,
opts,
versions,
}: {
path: string;
name: string;
opts: typeof options;
versions: typeof STORE_VERSIONS;
}) => {
const projectId = `project-${Date.now()}`;
const project: TestProject = {
id: projectId,
name: name,
path: path,
lastOpened: new Date().toISOString(),
};
const allProjects = [project, ...(opts?.additionalProjects || [])];
const currentProject = opts?.setAsCurrent !== false ? project : null;
const appState = {
state: {
projects: allProjects,
currentProject: currentProject,
currentView: currentProject ? 'board' : 'welcome',
theme: 'dark',
sidebarOpen: true,
apiKeys: { anthropic: '', google: '' },
chatSessions: [],
chatHistoryOpen: false,
maxConcurrency: 3,
},
version: versions.APP_STORE,
};
localStorage.setItem('automaker-storage', JSON.stringify(appState));
// Mark setup as complete
const setupState = {
state: {
isFirstRun: false,
setupComplete: true,
skipClaudeSetup: false,
},
version: versions.SETUP_STORE,
};
localStorage.setItem('automaker-setup', JSON.stringify(setupState));
},
{ path: projectPath, name: projectName, opts: options, versions: STORE_VERSIONS }
);
}
/**
* Set up a mock project in localStorage to bypass the welcome screen
* This simulates having opened a project before
@@ -595,7 +754,7 @@ export async function setupFirstRun(page: Page): Promise<void> {
* Set up the app to skip the setup wizard (setup already complete)
*/
export async function setupComplete(page: Page): Promise<void> {
await page.addInitScript(() => {
await page.addInitScript((versions: typeof STORE_VERSIONS) => {
// Mark setup as complete
const setupState = {
state: {
@@ -604,11 +763,11 @@ export async function setupComplete(page: Page): Promise<void> {
currentStep: 'complete',
skipClaudeSetup: false,
},
version: 2, // Must match app-store.ts persist version
version: versions.SETUP_STORE,
};
localStorage.setItem('automaker-setup', JSON.stringify(setupState));
});
}, STORE_VERSIONS);
}
/**