embed browser windows

This commit is contained in:
Auto
2026-04-01 11:28:51 +02:00
parent cfcba65de2
commit 9f87f7a314
12 changed files with 742 additions and 55 deletions

View File

@@ -436,6 +436,7 @@ function App() {
orchestratorStatus={wsState.orchestratorStatus}
recentActivity={wsState.recentActivity}
getAgentLogs={wsState.getAgentLogs}
browserScreenshots={wsState.browserScreenshots}
/>
@@ -573,6 +574,9 @@ function App() {
projectName={selectedProject}
activeTab={debugActiveTab}
onTabChange={setDebugActiveTab}
browserScreenshots={wsState.browserScreenshots}
onSubscribeBrowserView={wsState.subscribeBrowserView}
onUnsubscribeBrowserView={wsState.unsubscribeBrowserView}
/>
)}

View File

@@ -1,8 +1,9 @@
import { MessageCircle, ScrollText, X, Copy, Check, Code, FlaskConical } from 'lucide-react'
import { MessageCircle, ScrollText, X, Copy, Check, Code, FlaskConical, Maximize2 } from 'lucide-react'
import { useState } from 'react'
import { createPortal } from 'react-dom'
import { AgentAvatar } from './AgentAvatar'
import type { ActiveAgent, AgentLogEntry, AgentType } from '../lib/types'
import type { ActiveAgent, AgentLogEntry, AgentType, BrowserScreenshot } from '../lib/types'
import { AGENT_MASCOTS } from '../lib/types'
import { Card, CardContent } from '@/components/ui/card'
import { Button } from '@/components/ui/button'
import { Badge } from '@/components/ui/badge'
@@ -10,6 +11,7 @@ import { Badge } from '@/components/ui/badge'
interface AgentCardProps {
agent: ActiveAgent
onShowLogs?: (agentIndex: number) => void
browserScreenshot?: BrowserScreenshot
}
// Get a friendly state description
@@ -69,14 +71,56 @@ function getAgentTypeBadge(agentType: AgentType): { label: string; className: st
}
}
export function AgentCard({ agent, onShowLogs }: AgentCardProps) {
export function AgentCard({ agent, onShowLogs, browserScreenshot }: AgentCardProps) {
const isActive = ['thinking', 'working', 'testing'].includes(agent.state)
const hasLogs = agent.logs && agent.logs.length > 0
const typeBadge = getAgentTypeBadge(agent.agentType || 'coding')
const TypeIcon = typeBadge.icon
const [screenshotExpanded, setScreenshotExpanded] = useState(false)
return (
<Card className={`min-w-[180px] max-w-[220px] py-3 ${isActive ? 'animate-pulse' : ''}`}>
<>
{/* Expanded screenshot overlay */}
{screenshotExpanded && browserScreenshot && createPortal(
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70"
onClick={() => setScreenshotExpanded(false)}
>
<div
className="relative max-w-[90vw] max-h-[90vh] bg-card border-2 border-border rounded-lg overflow-hidden"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center justify-between px-3 py-2 bg-muted border-b border-border">
<div className="flex items-center gap-2">
<TypeIcon size={14} className={agent.agentType === 'testing' ? 'text-purple-500' : 'text-blue-500'} />
<span className="font-mono text-sm font-bold">
{AGENT_MASCOTS[agent.agentIndex % AGENT_MASCOTS.length]}
</span>
<Badge variant="outline" className="text-[10px] h-4">
{agent.agentType || 'coding'}
</Badge>
<span className="text-xs text-muted-foreground truncate">
{agent.featureName}
</span>
</div>
<button
onClick={() => setScreenshotExpanded(false)}
className="p-1 hover:bg-accent rounded transition-colors cursor-pointer"
>
<X size={16} />
</button>
</div>
<img
src={browserScreenshot.imageDataUrl}
alt={`Browser view - ${agent.featureName}`}
className="max-w-full max-h-[calc(90vh-3rem)] object-contain"
/>
</div>
</div>,
document.body
)}
<Card className={`min-w-[180px] ${browserScreenshot ? 'max-w-[280px]' : 'max-w-[220px]'} py-3 ${isActive ? 'animate-pulse' : ''}`}>
<CardContent className="p-3 space-y-2">
{/* Agent type badge */}
<div className="flex justify-end">
@@ -133,6 +177,29 @@ export function AgentCard({ agent, onShowLogs }: AgentCardProps) {
)}
</div>
{/* Browser screenshot thumbnail with expand */}
{browserScreenshot && (
<div className="pt-1 relative group">
<div
className="cursor-pointer overflow-hidden rounded border border-border/50"
onClick={() => setScreenshotExpanded(true)}
>
<img
src={browserScreenshot.imageDataUrl}
alt="Browser view"
className="w-full h-auto max-h-[120px] object-cover object-top"
/>
</div>
<button
onClick={() => setScreenshotExpanded(true)}
className="absolute top-2.5 right-1.5 p-0.5 bg-black/50 hover:bg-black/70 rounded transition-opacity opacity-0 group-hover:opacity-100 cursor-pointer"
title="Expand screenshot"
>
<Maximize2 size={12} className="text-white" />
</button>
</div>
)}
{/* Thought bubble */}
{agent.thought && (
<div className="pt-2 border-t border-border/50">
@@ -149,6 +216,7 @@ export function AgentCard({ agent, onShowLogs }: AgentCardProps) {
)}
</CardContent>
</Card>
</>
)
}

View File

@@ -3,7 +3,7 @@ import { useState } from 'react'
import { AgentCard, AgentLogModal } from './AgentCard'
import { ActivityFeed } from './ActivityFeed'
import { OrchestratorStatusCard } from './OrchestratorStatusCard'
import type { ActiveAgent, AgentLogEntry, OrchestratorStatus } from '../lib/types'
import type { ActiveAgent, AgentLogEntry, BrowserScreenshot, OrchestratorStatus } from '../lib/types'
import { Card, CardContent } from '@/components/ui/card'
import { Badge } from '@/components/ui/badge'
import { Button } from '@/components/ui/button'
@@ -21,6 +21,7 @@ interface AgentMissionControlProps {
}>
isExpanded?: boolean
getAgentLogs?: (agentIndex: number) => AgentLogEntry[]
browserScreenshots?: Map<string, BrowserScreenshot>
}
export function AgentMissionControl({
@@ -29,6 +30,7 @@ export function AgentMissionControl({
recentActivity,
isExpanded: defaultExpanded = true,
getAgentLogs,
browserScreenshots,
}: AgentMissionControlProps) {
const [isExpanded, setIsExpanded] = useState(defaultExpanded)
const [activityCollapsed, setActivityCollapsed] = useState(() => {
@@ -105,18 +107,25 @@ export function AgentMissionControl({
{/* Agent Cards Row */}
{agents.length > 0 && (
<div className="flex gap-4 overflow-x-auto pb-4">
{agents.map((agent) => (
<AgentCard
key={`agent-${agent.agentIndex}`}
agent={agent}
onShowLogs={(agentIndex) => {
const agentToShow = agents.find(a => a.agentIndex === agentIndex)
if (agentToShow) {
setSelectedAgentForLogs(agentToShow)
}
}}
/>
))}
{agents.map((agent) => {
// Find browser screenshot for this agent by matching agentIndex
const screenshot = browserScreenshots
? Array.from(browserScreenshots.values()).find(s => s.agentIndex === agent.agentIndex)
: undefined
return (
<AgentCard
key={`agent-${agent.agentIndex}`}
agent={agent}
onShowLogs={(agentIndex) => {
const agentToShow = agents.find(a => a.agentIndex === agentIndex)
if (agentToShow) {
setSelectedAgentForLogs(agentToShow)
}
}}
browserScreenshot={screenshot}
/>
)
})}
</div>
)}

View File

@@ -0,0 +1,150 @@
/**
* Browser View Panel
*
* Displays live screenshots from each agent's browser session.
* Subscribes to screenshot streaming on mount, unsubscribes on unmount.
*/
import { useEffect, useState } from 'react'
import { Monitor, X, Maximize2, Code, FlaskConical } from 'lucide-react'
import { Badge } from '@/components/ui/badge'
import { AGENT_MASCOTS } from '@/lib/types'
import type { BrowserScreenshot } from '@/lib/types'
interface BrowserViewPanelProps {
screenshots: Map<string, BrowserScreenshot>
onSubscribe: () => void
onUnsubscribe: () => void
}
export function BrowserViewPanel({ screenshots, onSubscribe, onUnsubscribe }: BrowserViewPanelProps) {
const [expandedSession, setExpandedSession] = useState<string | null>(null)
// Subscribe on mount, unsubscribe on unmount
useEffect(() => {
onSubscribe()
return () => onUnsubscribe()
}, [onSubscribe, onUnsubscribe])
const screenshotList = Array.from(screenshots.values())
if (screenshotList.length === 0) {
return (
<div className="flex flex-col items-center justify-center h-full text-muted-foreground gap-2">
<Monitor size={24} />
<span className="text-sm">No active browser sessions</span>
<span className="text-xs">Screenshots will appear when agents open browsers</span>
</div>
)
}
const expanded = expandedSession ? screenshots.get(expandedSession) : null
return (
<div className="h-full overflow-auto p-3">
{/* Expanded overlay */}
{expanded && (
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70"
onClick={() => setExpandedSession(null)}
>
<div
className="relative max-w-[90vw] max-h-[90vh] bg-card border-2 border-border rounded-lg overflow-hidden"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center justify-between px-3 py-2 bg-muted border-b border-border">
<div className="flex items-center gap-2">
{expanded.agentType === 'coding' ? (
<Code size={14} className="text-blue-500" />
) : (
<FlaskConical size={14} className="text-purple-500" />
)}
<span className="font-mono text-sm font-bold">
{AGENT_MASCOTS[expanded.agentIndex % AGENT_MASCOTS.length]}
</span>
<Badge variant="outline" className="text-[10px] h-4">
{expanded.agentType}
</Badge>
<span className="text-xs text-muted-foreground truncate">
{expanded.featureName}
</span>
</div>
<button
onClick={() => setExpandedSession(null)}
className="p-1 hover:bg-accent rounded transition-colors cursor-pointer"
>
<X size={16} />
</button>
</div>
<img
src={expanded.imageDataUrl}
alt={`Browser view - ${expanded.featureName}`}
className="max-w-full max-h-[calc(90vh-3rem)] object-contain"
/>
</div>
</div>
)}
{/* Screenshot grid — responsive 1/2/3 columns */}
<div className="grid gap-3 grid-cols-1 md:grid-cols-2 xl:grid-cols-3">
{screenshotList.map((screenshot) => (
<div
key={screenshot.sessionName}
className="border-2 border-border rounded-lg overflow-hidden bg-card hover:border-foreground/30 transition-colors"
>
{/* Card header */}
<div className="flex items-center justify-between px-2.5 py-1.5 bg-muted border-b border-border">
<div className="flex items-center gap-2 min-w-0">
{screenshot.agentType === 'coding' ? (
<Code size={12} className="text-blue-500 shrink-0" />
) : (
<FlaskConical size={12} className="text-purple-500 shrink-0" />
)}
<span className="font-mono text-xs font-bold">
{AGENT_MASCOTS[screenshot.agentIndex % AGENT_MASCOTS.length]}
</span>
<Badge variant="outline" className="text-[9px] h-3.5 px-1">
{screenshot.agentType}
</Badge>
<span className="text-[11px] text-muted-foreground truncate">
{screenshot.featureName}
</span>
</div>
<button
onClick={() => setExpandedSession(screenshot.sessionName)}
className="p-0.5 hover:bg-accent rounded transition-colors cursor-pointer shrink-0"
title="Expand"
>
<Maximize2 size={12} className="text-muted-foreground" />
</button>
</div>
{/* Screenshot image — capped height for compact grid */}
<div
className="cursor-pointer overflow-hidden"
onClick={() => setExpandedSession(screenshot.sessionName)}
>
<img
src={screenshot.imageDataUrl}
alt={`Browser - ${screenshot.featureName}`}
className="w-full h-auto max-h-[280px] object-cover object-top"
/>
</div>
{/* Timestamp footer */}
<div className="px-2.5 py-1 bg-muted border-t border-border">
<span className="text-[10px] text-muted-foreground font-mono">
{new Date(screenshot.timestamp).toLocaleTimeString('en-US', {
hour12: false,
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
})}
</span>
</div>
</div>
))}
</div>
</div>
)
}

View File

@@ -7,11 +7,12 @@
*/
import { useEffect, useRef, useState, useCallback } from 'react'
import { ChevronUp, ChevronDown, Trash2, Terminal as TerminalIcon, GripHorizontal, Cpu, Server } from 'lucide-react'
import { ChevronUp, ChevronDown, Trash2, Terminal as TerminalIcon, GripHorizontal, Cpu, Server, Monitor } from 'lucide-react'
import { Terminal } from './Terminal'
import { TerminalTabs } from './TerminalTabs'
import { BrowserViewPanel } from './BrowserViewPanel'
import { listTerminals, createTerminal, renameTerminal, deleteTerminal } from '@/lib/api'
import type { TerminalInfo } from '@/lib/types'
import type { TerminalInfo, BrowserScreenshot } from '@/lib/types'
import { Button } from '@/components/ui/button'
import { Badge } from '@/components/ui/badge'
@@ -21,7 +22,7 @@ const DEFAULT_HEIGHT = 288
const STORAGE_KEY = 'debug-panel-height'
const TAB_STORAGE_KEY = 'debug-panel-tab'
type TabType = 'agent' | 'devserver' | 'terminal'
type TabType = 'agent' | 'devserver' | 'terminal' | 'browsers'
interface DebugLogViewerProps {
logs: Array<{ line: string; timestamp: string }>
@@ -34,6 +35,9 @@ interface DebugLogViewerProps {
projectName: string
activeTab?: TabType
onTabChange?: (tab: TabType) => void
browserScreenshots?: Map<string, BrowserScreenshot>
onSubscribeBrowserView?: () => void
onUnsubscribeBrowserView?: () => void
}
type LogLevel = 'error' | 'warn' | 'debug' | 'info'
@@ -49,6 +53,9 @@ export function DebugLogViewer({
projectName,
activeTab: controlledActiveTab,
onTabChange,
browserScreenshots,
onSubscribeBrowserView,
onUnsubscribeBrowserView,
}: DebugLogViewerProps) {
const scrollRef = useRef<HTMLDivElement>(null)
const devScrollRef = useRef<HTMLDivElement>(null)
@@ -395,11 +402,28 @@ export function DebugLogViewer({
T
</Badge>
</Button>
<Button
variant={activeTab === 'browsers' ? 'secondary' : 'ghost'}
size="sm"
onClick={(e: React.MouseEvent) => {
e.stopPropagation()
setActiveTab('browsers')
}}
className="h-7 text-xs font-mono gap-1.5"
>
<Monitor size={12} />
Browsers
{browserScreenshots && browserScreenshots.size > 0 && (
<Badge variant="default" className="h-4 px-1.5 text-[10px]">
{browserScreenshots.size}
</Badge>
)}
</Button>
</div>
)}
{/* Log count and status - only for log tabs */}
{isOpen && activeTab !== 'terminal' && (
{isOpen && activeTab !== 'terminal' && activeTab !== 'browsers' && (
<>
{getCurrentLogCount() > 0 && (
<Badge variant="secondary" className="ml-2 font-mono">
@@ -417,7 +441,7 @@ export function DebugLogViewer({
<div className="flex items-center gap-2">
{/* Clear button - only for log tabs */}
{isOpen && activeTab !== 'terminal' && (
{isOpen && activeTab !== 'terminal' && activeTab !== 'browsers' && (
<Button
variant="ghost"
size="icon"
@@ -576,6 +600,15 @@ export function DebugLogViewer({
</div>
</div>
)}
{/* Browsers Tab */}
{activeTab === 'browsers' && browserScreenshots && onSubscribeBrowserView && onUnsubscribeBrowserView && (
<BrowserViewPanel
screenshots={browserScreenshots}
onSubscribe={onSubscribeBrowserView}
onUnsubscribe={onUnsubscribeBrowserView}
/>
)}
</div>
)}
</div>

View File

@@ -406,24 +406,6 @@ export function SettingsModal({ isOpen, onClose }: SettingsModalProps) {
/>
</div>
{/* Headless Browser Toggle */}
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="playwright-headless" className="font-medium">
Headless Browser
</Label>
<p className="text-sm text-muted-foreground">
Run browser without visible window (saves CPU)
</p>
</div>
<Switch
id="playwright-headless"
checked={settings.playwright_headless}
onCheckedChange={() => updateSettings.mutate({ playwright_headless: !settings.playwright_headless })}
disabled={isSaving}
/>
</div>
{/* Regression Agents */}
<div className="space-y-2">
<Label className="font-medium">Regression Agents</Label>

View File

@@ -10,6 +10,7 @@ import type {
ActiveAgent,
AgentMascot,
AgentLogEntry,
BrowserScreenshot,
OrchestratorStatus,
OrchestratorEvent,
} from '../lib/types'
@@ -53,6 +54,9 @@ interface WebSocketState {
celebration: CelebrationTrigger | null
// Orchestrator state for Mission Control
orchestratorStatus: OrchestratorStatus | null
// Browser view screenshots (sessionName -> latest screenshot)
browserScreenshots: Map<string, BrowserScreenshot>
browserViewSubscribed: boolean
}
const MAX_LOGS = 100 // Keep last 100 log lines
@@ -74,6 +78,8 @@ export function useProjectWebSocket(projectName: string | null) {
celebrationQueue: [],
celebration: null,
orchestratorStatus: null,
browserScreenshots: new Map(),
browserViewSubscribed: false,
})
const wsRef = useRef<WebSocket | null>(null)
@@ -119,11 +125,12 @@ export function useProjectWebSocket(projectName: string | null) {
setState(prev => ({
...prev,
agentStatus: message.status,
// Clear active agents and orchestrator status when process stops OR crashes to prevent stale UI
// Clear active agents, orchestrator status, and browser screenshots when process stops OR crashes
...((message.status === 'stopped' || message.status === 'crashed') && {
activeAgents: [],
recentActivity: [],
orchestratorStatus: null,
browserScreenshots: new Map(),
}),
}))
break
@@ -328,6 +335,22 @@ export function useProjectWebSocket(projectName: string | null) {
}))
break
case 'browser_screenshot':
setState(prev => {
const newScreenshots = new Map(prev.browserScreenshots)
newScreenshots.set(message.sessionName, {
sessionName: message.sessionName,
agentIndex: message.agentIndex,
agentType: message.agentType,
featureId: message.featureId,
featureName: message.featureName,
imageDataUrl: `data:image/png;base64,${message.imageData}`,
timestamp: message.timestamp,
})
return { ...prev, browserScreenshots: newScreenshots }
})
break
case 'pong':
// Heartbeat response
break
@@ -400,6 +423,8 @@ export function useProjectWebSocket(projectName: string | null) {
celebrationQueue: [],
celebration: null,
orchestratorStatus: null,
browserScreenshots: new Map(),
browserViewSubscribed: false,
})
if (!projectName) {
@@ -473,6 +498,22 @@ export function useProjectWebSocket(projectName: string | null) {
})
}, [])
// Subscribe to browser view screenshots
const subscribeBrowserView = useCallback(() => {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'browser_view_subscribe' }))
setState(prev => ({ ...prev, browserViewSubscribed: true }))
}
}, [])
// Unsubscribe from browser view screenshots
const unsubscribeBrowserView = useCallback(() => {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'browser_view_unsubscribe' }))
setState(prev => ({ ...prev, browserViewSubscribed: false }))
}
}, [])
return {
...state,
clearLogs,
@@ -480,5 +521,7 @@ export function useProjectWebSocket(projectName: string | null) {
clearCelebration,
getAgentLogs,
clearAgentLogs,
subscribeBrowserView,
unsubscribeBrowserView,
}
}

View File

@@ -266,7 +266,7 @@ export interface OrchestratorStatus {
}
// WebSocket message types
export type WSMessageType = 'progress' | 'feature_update' | 'log' | 'agent_status' | 'pong' | 'dev_log' | 'dev_server_status' | 'agent_update' | 'orchestrator_update'
export type WSMessageType = 'progress' | 'feature_update' | 'log' | 'agent_status' | 'pong' | 'dev_log' | 'dev_server_status' | 'agent_update' | 'orchestrator_update' | 'browser_screenshot'
export interface WSProgressMessage {
type: 'progress'
@@ -342,6 +342,28 @@ export interface WSOrchestratorUpdateMessage {
featureName?: string
}
export interface WSBrowserScreenshotMessage {
type: 'browser_screenshot'
sessionName: string
agentIndex: number
agentType: AgentType
featureId: number
featureName: string
imageData: string // base64 PNG
timestamp: string
}
// Browser screenshot stored in UI state
export interface BrowserScreenshot {
sessionName: string
agentIndex: number
agentType: AgentType
featureId: number
featureName: string
imageDataUrl: string // "data:image/png;base64,..."
timestamp: string
}
export type WSMessage =
| WSProgressMessage
| WSFeatureUpdateMessage
@@ -352,6 +374,7 @@ export type WSMessage =
| WSDevLogMessage
| WSDevServerStatusMessage
| WSOrchestratorUpdateMessage
| WSBrowserScreenshotMessage
// ============================================================================
// Spec Chat Types
@@ -636,7 +659,6 @@ export interface SettingsUpdate {
yolo_mode?: boolean
model?: string
testing_agent_ratio?: number
playwright_headless?: boolean
batch_size?: number
testing_batch_size?: number
api_provider?: string