chore: serialize session entries for tool calls and user actions (#803)
This commit is contained in:
@@ -57,16 +57,17 @@ export class BrowserServerBackend implements ServerBackend {
|
|||||||
const context = this._context!;
|
const context = this._context!;
|
||||||
const response = new Response(context, schema.name, parsedArguments);
|
const response = new Response(context, schema.name, parsedArguments);
|
||||||
const tool = this._tools.find(tool => tool.schema.name === schema.name)!;
|
const tool = this._tools.find(tool => tool.schema.name === schema.name)!;
|
||||||
await context.setInputRecorderEnabled(false);
|
context.setRunningTool(true);
|
||||||
try {
|
try {
|
||||||
await tool.handle(context, parsedArguments, response);
|
await tool.handle(context, parsedArguments, response);
|
||||||
} catch (error) {
|
await response.finish();
|
||||||
|
this._sessionLog?.logResponse(response);
|
||||||
|
} catch (error: any) {
|
||||||
response.addError(String(error));
|
response.addError(String(error));
|
||||||
} finally {
|
} finally {
|
||||||
await context.setInputRecorderEnabled(true);
|
context.setRunningTool(false);
|
||||||
}
|
}
|
||||||
await this._sessionLog?.logResponse(response);
|
return response.serialize();
|
||||||
return await response.serialize();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
serverInitialized(version: mcpServer.ClientVersion | undefined) {
|
serverInitialized(version: mcpServer.ClientVersion | undefined) {
|
||||||
|
|||||||
112
src/context.ts
112
src/context.ts
@@ -24,13 +24,14 @@ import type { Tool } from './tools/tool.js';
|
|||||||
import type { FullConfig } from './config.js';
|
import type { FullConfig } from './config.js';
|
||||||
import type { BrowserContextFactory } from './browserContextFactory.js';
|
import type { BrowserContextFactory } from './browserContextFactory.js';
|
||||||
import type * as actions from './actions.js';
|
import type * as actions from './actions.js';
|
||||||
import type { Action, SessionLog } from './sessionLog.js';
|
import type { SessionLog } from './sessionLog.js';
|
||||||
|
|
||||||
const testDebug = debug('pw:mcp:test');
|
const testDebug = debug('pw:mcp:test');
|
||||||
|
|
||||||
export class Context {
|
export class Context {
|
||||||
readonly tools: Tool[];
|
readonly tools: Tool[];
|
||||||
readonly config: FullConfig;
|
readonly config: FullConfig;
|
||||||
|
readonly sessionLog: SessionLog | undefined;
|
||||||
private _browserContextPromise: Promise<{ browserContext: playwright.BrowserContext, close: () => Promise<void> }> | undefined;
|
private _browserContextPromise: Promise<{ browserContext: playwright.BrowserContext, close: () => Promise<void> }> | undefined;
|
||||||
private _browserContextFactory: BrowserContextFactory;
|
private _browserContextFactory: BrowserContextFactory;
|
||||||
private _tabs: Tab[] = [];
|
private _tabs: Tab[] = [];
|
||||||
@@ -40,14 +41,13 @@ export class Context {
|
|||||||
|
|
||||||
private static _allContexts: Set<Context> = new Set();
|
private static _allContexts: Set<Context> = new Set();
|
||||||
private _closeBrowserContextPromise: Promise<void> | undefined;
|
private _closeBrowserContextPromise: Promise<void> | undefined;
|
||||||
private _inputRecorder: InputRecorder | undefined;
|
private _isRunningTool: boolean = false;
|
||||||
private _sessionLog: SessionLog | undefined;
|
|
||||||
|
|
||||||
constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory, sessionLog: SessionLog | undefined) {
|
constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory, sessionLog: SessionLog | undefined) {
|
||||||
this.tools = tools;
|
this.tools = tools;
|
||||||
this.config = config;
|
this.config = config;
|
||||||
this._browserContextFactory = browserContextFactory;
|
this._browserContextFactory = browserContextFactory;
|
||||||
this._sessionLog = sessionLog;
|
this.sessionLog = sessionLog;
|
||||||
testDebug('create context');
|
testDebug('create context');
|
||||||
Context._allContexts.add(this);
|
Context._allContexts.add(this);
|
||||||
}
|
}
|
||||||
@@ -93,29 +93,6 @@ export class Context {
|
|||||||
return this._currentTab!;
|
return this._currentTab!;
|
||||||
}
|
}
|
||||||
|
|
||||||
async listTabsMarkdown(force: boolean = false): Promise<string[]> {
|
|
||||||
if (this._tabs.length === 1 && !force)
|
|
||||||
return [];
|
|
||||||
|
|
||||||
if (!this._tabs.length) {
|
|
||||||
return [
|
|
||||||
'### Open tabs',
|
|
||||||
'No open tabs. Use the "browser_navigate" tool to navigate to a page first.',
|
|
||||||
'',
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
const lines: string[] = ['### Open tabs'];
|
|
||||||
for (let i = 0; i < this._tabs.length; i++) {
|
|
||||||
const tab = this._tabs[i];
|
|
||||||
const title = await tab.title();
|
|
||||||
const url = tab.page.url();
|
|
||||||
const current = tab === this._currentTab ? ' (current)' : '';
|
|
||||||
lines.push(`- ${i}:${current} [${title}] (${url})`);
|
|
||||||
}
|
|
||||||
lines.push('');
|
|
||||||
return lines;
|
|
||||||
}
|
|
||||||
|
|
||||||
async closeTab(index: number | undefined): Promise<string> {
|
async closeTab(index: number | undefined): Promise<string> {
|
||||||
const tab = index === undefined ? this._currentTab : this._tabs[index];
|
const tab = index === undefined ? this._currentTab : this._tabs[index];
|
||||||
@@ -152,8 +129,12 @@ export class Context {
|
|||||||
this._closeBrowserContextPromise = undefined;
|
this._closeBrowserContextPromise = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
async setInputRecorderEnabled(enabled: boolean) {
|
isRunningTool() {
|
||||||
await this._inputRecorder?.setEnabled(enabled);
|
return this._isRunningTool;
|
||||||
|
}
|
||||||
|
|
||||||
|
setRunningTool(isRunningTool: boolean) {
|
||||||
|
this._isRunningTool = isRunningTool;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async _closeBrowserContextImpl() {
|
private async _closeBrowserContextImpl() {
|
||||||
@@ -208,8 +189,8 @@ export class Context {
|
|||||||
const result = await this._browserContextFactory.createContext(this.clientVersion!);
|
const result = await this._browserContextFactory.createContext(this.clientVersion!);
|
||||||
const { browserContext } = result;
|
const { browserContext } = result;
|
||||||
await this._setupRequestInterception(browserContext);
|
await this._setupRequestInterception(browserContext);
|
||||||
if (this._sessionLog)
|
if (this.sessionLog)
|
||||||
this._inputRecorder = await InputRecorder.create(this._sessionLog, browserContext);
|
await InputRecorder.create(this, browserContext);
|
||||||
for (const page of browserContext.pages())
|
for (const page of browserContext.pages())
|
||||||
this._onPageCreated(page);
|
this._onPageCreated(page);
|
||||||
browserContext.on('page', page => this._onPageCreated(page));
|
browserContext.on('page', page => this._onPageCreated(page));
|
||||||
@@ -226,87 +207,54 @@ export class Context {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class InputRecorder {
|
export class InputRecorder {
|
||||||
private _actions: Action[] = [];
|
private _context: Context;
|
||||||
private _enabled = false;
|
|
||||||
private _sessionLog: SessionLog;
|
|
||||||
private _browserContext: playwright.BrowserContext;
|
private _browserContext: playwright.BrowserContext;
|
||||||
private _flushTimer: NodeJS.Timeout | undefined;
|
|
||||||
|
|
||||||
private constructor(sessionLog: SessionLog, browserContext: playwright.BrowserContext) {
|
private constructor(context: Context, browserContext: playwright.BrowserContext) {
|
||||||
this._sessionLog = sessionLog;
|
this._context = context;
|
||||||
this._browserContext = browserContext;
|
this._browserContext = browserContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
static async create(sessionLog: SessionLog, browserContext: playwright.BrowserContext) {
|
static async create(context: Context, browserContext: playwright.BrowserContext) {
|
||||||
const recorder = new InputRecorder(sessionLog, browserContext);
|
const recorder = new InputRecorder(context, browserContext);
|
||||||
await recorder._initialize();
|
await recorder._initialize();
|
||||||
await recorder.setEnabled(true);
|
|
||||||
return recorder;
|
return recorder;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async _initialize() {
|
private async _initialize() {
|
||||||
|
const sessionLog = this._context.sessionLog!;
|
||||||
await (this._browserContext as any)._enableRecorder({
|
await (this._browserContext as any)._enableRecorder({
|
||||||
mode: 'recording',
|
mode: 'recording',
|
||||||
recorderMode: 'api',
|
recorderMode: 'api',
|
||||||
}, {
|
}, {
|
||||||
actionAdded: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
actionAdded: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
||||||
if (!this._enabled)
|
if (this._context.isRunningTool())
|
||||||
return;
|
return;
|
||||||
const tab = Tab.forPage(page);
|
const tab = Tab.forPage(page);
|
||||||
this._actions.push({ ...data, tab, code: code.trim(), timestamp: performance.now() });
|
if (tab)
|
||||||
this._scheduleFlush();
|
sessionLog.logUserAction(data.action, tab, code, false);
|
||||||
},
|
},
|
||||||
actionUpdated: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
actionUpdated: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
||||||
if (!this._enabled)
|
if (this._context.isRunningTool())
|
||||||
return;
|
return;
|
||||||
const tab = Tab.forPage(page);
|
const tab = Tab.forPage(page);
|
||||||
this._actions[this._actions.length - 1] = { ...data, tab, code: code.trim(), timestamp: performance.now() };
|
if (tab)
|
||||||
this._scheduleFlush();
|
sessionLog.logUserAction(data.action, tab, code, true);
|
||||||
},
|
},
|
||||||
signalAdded: (page: playwright.Page, data: actions.SignalInContext) => {
|
signalAdded: (page: playwright.Page, data: actions.SignalInContext) => {
|
||||||
|
if (this._context.isRunningTool())
|
||||||
|
return;
|
||||||
if (data.signal.name !== 'navigation')
|
if (data.signal.name !== 'navigation')
|
||||||
return;
|
return;
|
||||||
const tab = Tab.forPage(page);
|
const tab = Tab.forPage(page);
|
||||||
this._actions.push({
|
const navigateAction: actions.Action = {
|
||||||
frame: data.frame,
|
|
||||||
action: {
|
|
||||||
name: 'navigate',
|
name: 'navigate',
|
||||||
url: data.signal.url,
|
url: data.signal.url,
|
||||||
signals: [],
|
signals: [],
|
||||||
},
|
};
|
||||||
startTime: data.timestamp,
|
if (tab)
|
||||||
endTime: data.timestamp,
|
sessionLog.logUserAction(navigateAction, tab, `await page.goto('${data.signal.url}');`, false);
|
||||||
tab,
|
|
||||||
code: `await page.goto('${data.signal.url}');`,
|
|
||||||
timestamp: performance.now(),
|
|
||||||
});
|
|
||||||
this._scheduleFlush();
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async setEnabled(enabled: boolean) {
|
|
||||||
this._enabled = enabled;
|
|
||||||
if (!enabled)
|
|
||||||
await this._flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
private _clearTimer() {
|
|
||||||
if (this._flushTimer) {
|
|
||||||
clearTimeout(this._flushTimer);
|
|
||||||
this._flushTimer = undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _scheduleFlush() {
|
|
||||||
this._clearTimer();
|
|
||||||
this._flushTimer = setTimeout(() => this._flush(), 1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _flush() {
|
|
||||||
this._clearTimer();
|
|
||||||
const actions = this._actions;
|
|
||||||
this._actions = [];
|
|
||||||
await this._sessionLog.logActions(actions);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,8 +16,7 @@
|
|||||||
|
|
||||||
import { renderModalStates } from './tab.js';
|
import { renderModalStates } from './tab.js';
|
||||||
|
|
||||||
import type { TabSnapshot } from './tab.js';
|
import type { Tab, TabSnapshot } from './tab.js';
|
||||||
import type { ModalState } from './tools/tool.js';
|
|
||||||
import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js';
|
import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js';
|
||||||
import type { Context } from './context.js';
|
import type { Context } from './context.js';
|
||||||
|
|
||||||
@@ -28,7 +27,7 @@ export class Response {
|
|||||||
private _context: Context;
|
private _context: Context;
|
||||||
private _includeSnapshot = false;
|
private _includeSnapshot = false;
|
||||||
private _includeTabs = false;
|
private _includeTabs = false;
|
||||||
private _snapshot: { tabSnapshot?: TabSnapshot, modalState?: ModalState } | undefined;
|
private _tabSnapshot: TabSnapshot | undefined;
|
||||||
|
|
||||||
readonly toolName: string;
|
readonly toolName: string;
|
||||||
readonly toolArgs: Record<string, any>;
|
readonly toolArgs: Record<string, any>;
|
||||||
@@ -81,17 +80,20 @@ export class Response {
|
|||||||
this._includeTabs = true;
|
this._includeTabs = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
async snapshot(): Promise<{ tabSnapshot?: TabSnapshot, modalState?: ModalState }> {
|
async finish() {
|
||||||
if (this._snapshot)
|
// All the async snapshotting post-action is happening here.
|
||||||
return this._snapshot;
|
// Everything below should race against modal states.
|
||||||
if (this._includeSnapshot && this._context.currentTab())
|
if (this._includeSnapshot && this._context.currentTab())
|
||||||
this._snapshot = await this._context.currentTabOrDie().captureSnapshot();
|
this._tabSnapshot = await this._context.currentTabOrDie().captureSnapshot();
|
||||||
else
|
for (const tab of this._context.tabs())
|
||||||
this._snapshot = {};
|
await tab.updateTitle();
|
||||||
return this._snapshot;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async serialize(): Promise<{ content: (TextContent | ImageContent)[], isError?: boolean }> {
|
tabSnapshot(): TabSnapshot | undefined {
|
||||||
|
return this._tabSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
serialize(): { content: (TextContent | ImageContent)[], isError?: boolean } {
|
||||||
const response: string[] = [];
|
const response: string[] = [];
|
||||||
|
|
||||||
// Start with command result.
|
// Start with command result.
|
||||||
@@ -112,16 +114,14 @@ ${this._code.join('\n')}
|
|||||||
|
|
||||||
// List browser tabs.
|
// List browser tabs.
|
||||||
if (this._includeSnapshot || this._includeTabs)
|
if (this._includeSnapshot || this._includeTabs)
|
||||||
response.push(...(await this._context.listTabsMarkdown(this._includeTabs)));
|
response.push(...renderTabsMarkdown(this._context.tabs(), this._includeTabs));
|
||||||
|
|
||||||
// Add snapshot if provided.
|
// Add snapshot if provided.
|
||||||
const snapshot = await this.snapshot();
|
if (this._tabSnapshot?.modalStates.length) {
|
||||||
if (snapshot?.modalState) {
|
response.push(...renderModalStates(this._context, this._tabSnapshot.modalStates));
|
||||||
response.push(...renderModalStates(this._context, [snapshot.modalState]));
|
|
||||||
response.push('');
|
response.push('');
|
||||||
}
|
} else if (this._tabSnapshot) {
|
||||||
if (snapshot?.tabSnapshot) {
|
response.push(renderTabSnapshot(this._tabSnapshot));
|
||||||
response.push(renderTabSnapshot(snapshot.tabSnapshot));
|
|
||||||
response.push('');
|
response.push('');
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,6 +172,28 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot): string {
|
|||||||
return lines.join('\n');
|
return lines.join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderTabsMarkdown(tabs: Tab[], force: boolean = false): string[] {
|
||||||
|
if (tabs.length === 1 && !force)
|
||||||
|
return [];
|
||||||
|
|
||||||
|
if (!tabs.length) {
|
||||||
|
return [
|
||||||
|
'### Open tabs',
|
||||||
|
'No open tabs. Use the "browser_navigate" tool to navigate to a page first.',
|
||||||
|
'',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
const lines: string[] = ['### Open tabs'];
|
||||||
|
for (let i = 0; i < tabs.length; i++) {
|
||||||
|
const tab = tabs[i];
|
||||||
|
const current = tab.isCurrentTab() ? ' (current)' : '';
|
||||||
|
lines.push(`- ${i}:${current} [${tab.lastTitle()}] (${tab.page.url()})`);
|
||||||
|
}
|
||||||
|
lines.push('');
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
|
||||||
function trim(text: string, maxLength: number) {
|
function trim(text: string, maxLength: number) {
|
||||||
if (text.length <= maxLength)
|
if (text.length <= maxLength)
|
||||||
return text;
|
return text;
|
||||||
|
|||||||
@@ -19,17 +19,32 @@ import path from 'path';
|
|||||||
|
|
||||||
import { outputFile } from './config.js';
|
import { outputFile } from './config.js';
|
||||||
import { Response } from './response.js';
|
import { Response } from './response.js';
|
||||||
|
|
||||||
|
import { logUnhandledError } from './log.js';
|
||||||
import type { FullConfig } from './config.js';
|
import type { FullConfig } from './config.js';
|
||||||
import type * as actions from './actions.js';
|
import type * as actions from './actions.js';
|
||||||
import type { Tab } from './tab.js';
|
import type { Tab, TabSnapshot } from './tab.js';
|
||||||
|
|
||||||
export type Action = actions.ActionInContext & { code: string; tab?: Tab | undefined; timestamp: number };
|
type LogEntry = {
|
||||||
|
timestamp: number;
|
||||||
|
toolCall?: {
|
||||||
|
toolName: string;
|
||||||
|
toolArgs: Record<string, any>;
|
||||||
|
result: string;
|
||||||
|
isError?: boolean;
|
||||||
|
};
|
||||||
|
userAction?: actions.Action;
|
||||||
|
code: string;
|
||||||
|
tabSnapshot?: TabSnapshot;
|
||||||
|
};
|
||||||
|
|
||||||
export class SessionLog {
|
export class SessionLog {
|
||||||
private _folder: string;
|
private _folder: string;
|
||||||
private _file: string;
|
private _file: string;
|
||||||
private _ordinal = 0;
|
private _ordinal = 0;
|
||||||
private _lastModified = 0;
|
private _pendingEntries: LogEntry[] = [];
|
||||||
|
private _sessionFileQueue = Promise.resolve();
|
||||||
|
private _flushEntriesTimeout: NodeJS.Timeout | undefined;
|
||||||
|
|
||||||
constructor(sessionFolder: string) {
|
constructor(sessionFolder: string) {
|
||||||
this._folder = sessionFolder;
|
this._folder = sessionFolder;
|
||||||
@@ -44,90 +59,118 @@ export class SessionLog {
|
|||||||
return new SessionLog(sessionFolder);
|
return new SessionLog(sessionFolder);
|
||||||
}
|
}
|
||||||
|
|
||||||
lastModified() {
|
logResponse(response: Response) {
|
||||||
return this._lastModified;
|
const entry: LogEntry = {
|
||||||
|
timestamp: performance.now(),
|
||||||
|
toolCall: {
|
||||||
|
toolName: response.toolName,
|
||||||
|
toolArgs: response.toolArgs,
|
||||||
|
result: response.result(),
|
||||||
|
isError: response.isError(),
|
||||||
|
},
|
||||||
|
code: response.code(),
|
||||||
|
tabSnapshot: response.tabSnapshot(),
|
||||||
|
};
|
||||||
|
this._appendEntry(entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
async logResponse(response: Response) {
|
logUserAction(action: actions.Action, tab: Tab, code: string, isUpdate: boolean) {
|
||||||
this._lastModified = performance.now();
|
code = code.trim();
|
||||||
const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`;
|
if (isUpdate) {
|
||||||
const lines: string[] = [
|
const lastEntry = this._pendingEntries[this._pendingEntries.length - 1];
|
||||||
`### Tool call: ${response.toolName}`,
|
if (lastEntry.userAction?.name === action.name) {
|
||||||
|
lastEntry.userAction = action;
|
||||||
|
lastEntry.code = code;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (action.name === 'navigate') {
|
||||||
|
// Already logged at this location.
|
||||||
|
const lastEntry = this._pendingEntries[this._pendingEntries.length - 1];
|
||||||
|
if (lastEntry?.tabSnapshot?.url === action.url)
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const entry: LogEntry = {
|
||||||
|
timestamp: performance.now(),
|
||||||
|
userAction: action,
|
||||||
|
code,
|
||||||
|
tabSnapshot: {
|
||||||
|
url: tab.page.url(),
|
||||||
|
title: '',
|
||||||
|
ariaSnapshot: action.ariaSnapshot || '',
|
||||||
|
modalStates: [],
|
||||||
|
consoleMessages: [],
|
||||||
|
downloads: [],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
this._appendEntry(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
private _appendEntry(entry: LogEntry) {
|
||||||
|
this._pendingEntries.push(entry);
|
||||||
|
if (this._flushEntriesTimeout)
|
||||||
|
clearTimeout(this._flushEntriesTimeout);
|
||||||
|
this._flushEntriesTimeout = setTimeout(() => this._flushEntries(), 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _flushEntries() {
|
||||||
|
clearTimeout(this._flushEntriesTimeout);
|
||||||
|
const entries = this._pendingEntries;
|
||||||
|
this._pendingEntries = [];
|
||||||
|
const lines: string[] = [''];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
const ordinal = (++this._ordinal).toString().padStart(3, '0');
|
||||||
|
if (entry.toolCall) {
|
||||||
|
lines.push(
|
||||||
|
`### Tool call: ${entry.toolCall.toolName}`,
|
||||||
`- Args`,
|
`- Args`,
|
||||||
'```json',
|
'```json',
|
||||||
JSON.stringify(response.toolArgs, null, 2),
|
JSON.stringify(entry.toolCall.toolArgs, null, 2),
|
||||||
'```',
|
'```',
|
||||||
];
|
|
||||||
if (response.result()) {
|
|
||||||
lines.push(
|
|
||||||
response.isError() ? `- Error` : `- Result`,
|
|
||||||
'```',
|
|
||||||
response.result(),
|
|
||||||
'```');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.code()) {
|
|
||||||
lines.push(
|
|
||||||
`- Code`,
|
|
||||||
'```js',
|
|
||||||
response.code(),
|
|
||||||
'```');
|
|
||||||
}
|
|
||||||
|
|
||||||
const snapshot = await response.snapshot();
|
|
||||||
if (snapshot?.tabSnapshot) {
|
|
||||||
const fileName = `${prefix}.snapshot.yml`;
|
|
||||||
await fs.promises.writeFile(path.join(this._folder, fileName), snapshot.tabSnapshot?.ariaSnapshot);
|
|
||||||
lines.push(`- Snapshot: ${fileName}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const image of response.images()) {
|
|
||||||
const fileName = `${prefix}.screenshot.${extension(image.contentType)}`;
|
|
||||||
await fs.promises.writeFile(path.join(this._folder, fileName), image.data);
|
|
||||||
lines.push(`- Screenshot: ${fileName}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push('', '', '');
|
|
||||||
await this._appendLines(lines);
|
|
||||||
}
|
|
||||||
|
|
||||||
async logActions(actions: Action[]) {
|
|
||||||
// Skip recent navigation, it is a side-effect of the previous action or tool use.
|
|
||||||
if (actions?.[0]?.action?.name === 'navigate' && actions[0].timestamp - this._lastModified < 1000)
|
|
||||||
return;
|
|
||||||
|
|
||||||
this._lastModified = performance.now();
|
|
||||||
const lines: string[] = [];
|
|
||||||
for (const action of actions) {
|
|
||||||
const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`;
|
|
||||||
lines.push(
|
|
||||||
`### User action: ${action.action.name}`,
|
|
||||||
);
|
);
|
||||||
if (action.code) {
|
if (entry.toolCall.result) {
|
||||||
|
lines.push(
|
||||||
|
entry.toolCall.isError ? `- Error` : `- Result`,
|
||||||
|
'```',
|
||||||
|
entry.toolCall.result,
|
||||||
|
'```',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.userAction) {
|
||||||
|
const actionData = { ...entry.userAction } as any;
|
||||||
|
delete actionData.ariaSnapshot;
|
||||||
|
delete actionData.selector;
|
||||||
|
delete actionData.signals;
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
`### User action: ${entry.userAction.name}`,
|
||||||
|
`- Args`,
|
||||||
|
'```json',
|
||||||
|
JSON.stringify(actionData, null, 2),
|
||||||
|
'```',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.code) {
|
||||||
lines.push(
|
lines.push(
|
||||||
`- Code`,
|
`- Code`,
|
||||||
'```js',
|
'```js',
|
||||||
action.code,
|
entry.code,
|
||||||
'```');
|
'```');
|
||||||
}
|
}
|
||||||
if (action.action.ariaSnapshot) {
|
|
||||||
const fileName = `${prefix}.snapshot.yml`;
|
if (entry.tabSnapshot) {
|
||||||
await fs.promises.writeFile(path.join(this._folder, fileName), action.action.ariaSnapshot);
|
const fileName = `${ordinal}.snapshot.yml`;
|
||||||
|
fs.promises.writeFile(path.join(this._folder, fileName), entry.tabSnapshot.ariaSnapshot).catch(logUnhandledError);
|
||||||
lines.push(`- Snapshot: ${fileName}`);
|
lines.push(`- Snapshot: ${fileName}`);
|
||||||
}
|
}
|
||||||
lines.push('', '', '');
|
|
||||||
|
lines.push('', '');
|
||||||
}
|
}
|
||||||
|
|
||||||
await this._appendLines(lines);
|
this._sessionFileQueue = this._sessionFileQueue.then(() => fs.promises.appendFile(this._file, lines.join('\n')));
|
||||||
}
|
|
||||||
|
|
||||||
private async _appendLines(lines: string[]) {
|
|
||||||
await fs.promises.appendFile(this._file, lines.join('\n'));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function extension(contentType: string): 'jpg' | 'png' {
|
|
||||||
if (contentType === 'image/jpeg')
|
|
||||||
return 'jpg';
|
|
||||||
return 'png';
|
|
||||||
}
|
|
||||||
|
|||||||
40
src/tab.ts
40
src/tab.ts
@@ -48,6 +48,7 @@ export type TabSnapshot = {
|
|||||||
export class Tab extends EventEmitter<TabEventsInterface> {
|
export class Tab extends EventEmitter<TabEventsInterface> {
|
||||||
readonly context: Context;
|
readonly context: Context;
|
||||||
readonly page: playwright.Page;
|
readonly page: playwright.Page;
|
||||||
|
private _lastTitle = 'about:blank';
|
||||||
private _consoleMessages: ConsoleMessage[] = [];
|
private _consoleMessages: ConsoleMessage[] = [];
|
||||||
private _recentConsoleMessages: ConsoleMessage[] = [];
|
private _recentConsoleMessages: ConsoleMessage[] = [];
|
||||||
private _requests: Map<playwright.Request, playwright.Response | null> = new Map();
|
private _requests: Map<playwright.Request, playwright.Response | null> = new Map();
|
||||||
@@ -137,8 +138,18 @@ export class Tab extends EventEmitter<TabEventsInterface> {
|
|||||||
this._onPageClose(this);
|
this._onPageClose(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
async title(): Promise<string> {
|
async updateTitle() {
|
||||||
return await callOnPageNoTrace(this.page, page => page.title());
|
await this._raceAgainstModalStates(async () => {
|
||||||
|
this._lastTitle = await callOnPageNoTrace(this.page, page => page.title());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
lastTitle(): string {
|
||||||
|
return this._lastTitle;
|
||||||
|
}
|
||||||
|
|
||||||
|
isCurrentTab(): boolean {
|
||||||
|
return this === this.context.currentTab();
|
||||||
}
|
}
|
||||||
|
|
||||||
async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {
|
async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {
|
||||||
@@ -182,15 +193,15 @@ export class Tab extends EventEmitter<TabEventsInterface> {
|
|||||||
return this._requests;
|
return this._requests;
|
||||||
}
|
}
|
||||||
|
|
||||||
async captureSnapshot(): Promise<{ tabSnapshot?: TabSnapshot, modalState?: ModalState }> {
|
async captureSnapshot(): Promise<TabSnapshot> {
|
||||||
let tabSnapshot: TabSnapshot | undefined;
|
let tabSnapshot: TabSnapshot | undefined;
|
||||||
const modalState = await this._raceAgainstModalStates(async () => {
|
const modalStates = await this._raceAgainstModalStates(async () => {
|
||||||
const snapshot = await (this.page as PageEx)._snapshotForAI();
|
const snapshot = await (this.page as PageEx)._snapshotForAI();
|
||||||
tabSnapshot = {
|
tabSnapshot = {
|
||||||
url: this.page.url(),
|
url: this.page.url(),
|
||||||
title: await this.page.title(),
|
title: await this.page.title(),
|
||||||
ariaSnapshot: snapshot,
|
ariaSnapshot: snapshot,
|
||||||
modalStates: this.modalStates(),
|
modalStates: [],
|
||||||
consoleMessages: [],
|
consoleMessages: [],
|
||||||
downloads: this._downloads,
|
downloads: this._downloads,
|
||||||
};
|
};
|
||||||
@@ -200,25 +211,32 @@ export class Tab extends EventEmitter<TabEventsInterface> {
|
|||||||
tabSnapshot.consoleMessages = this._recentConsoleMessages;
|
tabSnapshot.consoleMessages = this._recentConsoleMessages;
|
||||||
this._recentConsoleMessages = [];
|
this._recentConsoleMessages = [];
|
||||||
}
|
}
|
||||||
return { tabSnapshot, modalState };
|
return tabSnapshot ?? {
|
||||||
|
url: this.page.url(),
|
||||||
|
title: '',
|
||||||
|
ariaSnapshot: '',
|
||||||
|
modalStates,
|
||||||
|
consoleMessages: [],
|
||||||
|
downloads: [],
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private _javaScriptBlocked(): boolean {
|
private _javaScriptBlocked(): boolean {
|
||||||
return this._modalStates.some(state => state.type === 'dialog');
|
return this._modalStates.some(state => state.type === 'dialog');
|
||||||
}
|
}
|
||||||
|
|
||||||
private async _raceAgainstModalStates(action: () => Promise<void>): Promise<ModalState | undefined> {
|
private async _raceAgainstModalStates(action: () => Promise<void>): Promise<ModalState[]> {
|
||||||
if (this.modalStates().length)
|
if (this.modalStates().length)
|
||||||
return this.modalStates()[0];
|
return this.modalStates();
|
||||||
|
|
||||||
const promise = new ManualPromise<ModalState>();
|
const promise = new ManualPromise<ModalState[]>();
|
||||||
const listener = (modalState: ModalState) => promise.resolve(modalState);
|
const listener = (modalState: ModalState) => promise.resolve([modalState]);
|
||||||
this.once(TabEvents.modalState, listener);
|
this.once(TabEvents.modalState, listener);
|
||||||
|
|
||||||
return await Promise.race([
|
return await Promise.race([
|
||||||
action().then(() => {
|
action().then(() => {
|
||||||
this.off(TabEvents.modalState, listener);
|
this.off(TabEvents.modalState, listener);
|
||||||
return undefined;
|
return [];
|
||||||
}),
|
}),
|
||||||
promise,
|
promise,
|
||||||
]);
|
]);
|
||||||
|
|||||||
@@ -47,8 +47,8 @@ test('session log should record tool calls', async ({ startClient, server }, tes
|
|||||||
|
|
||||||
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||||
const sessionFolder = output.substring('Session: '.length);
|
const sessionFolder = output.substring('Session: '.length);
|
||||||
const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
|
await expect.poll(() => readSessionLog(sessionFolder)).toBe(`
|
||||||
expect(sessionLog).toBe(`### Tool call: browser_navigate
|
### Tool call: browser_navigate
|
||||||
- Args
|
- Args
|
||||||
\`\`\`json
|
\`\`\`json
|
||||||
{
|
{
|
||||||
@@ -76,11 +76,120 @@ await page.getByRole('button', { name: 'Submit' }).click();
|
|||||||
\`\`\`
|
\`\`\`
|
||||||
- Snapshot: 002.snapshot.yml
|
- Snapshot: 002.snapshot.yml
|
||||||
|
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('session log should record user action', async ({ cdpServer, startClient }, testInfo) => {
|
||||||
|
const browserContext = await cdpServer.start();
|
||||||
|
const { client, stderr } = await startClient({
|
||||||
|
args: [
|
||||||
|
'--save-session',
|
||||||
|
'--output-dir', testInfo.outputPath('output'),
|
||||||
|
`--cdp-endpoint=${cdpServer.endpoint}`,
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
// Force browser context creation.
|
||||||
|
await client.callTool({
|
||||||
|
name: 'browser_snapshot',
|
||||||
|
});
|
||||||
|
|
||||||
|
const [page] = browserContext.pages();
|
||||||
|
await page.setContent(`
|
||||||
|
<button>Button 1</button>
|
||||||
|
<button>Button 2</button>
|
||||||
|
`);
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: 'Button 1' }).click();
|
||||||
|
|
||||||
|
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||||
|
const sessionFolder = output.substring('Session: '.length);
|
||||||
|
|
||||||
|
await expect.poll(() => readSessionLog(sessionFolder)).toBe(`
|
||||||
|
### Tool call: browser_snapshot
|
||||||
|
- Args
|
||||||
|
\`\`\`json
|
||||||
|
{}
|
||||||
|
\`\`\`
|
||||||
|
- Snapshot: 001.snapshot.yml
|
||||||
|
|
||||||
|
|
||||||
|
### User action: click
|
||||||
|
- Args
|
||||||
|
\`\`\`json
|
||||||
|
{
|
||||||
|
"name": "click",
|
||||||
|
"ref": "e2",
|
||||||
|
"button": "left",
|
||||||
|
"modifiers": 0,
|
||||||
|
"clickCount": 1
|
||||||
|
}
|
||||||
|
\`\`\`
|
||||||
|
- Code
|
||||||
|
\`\`\`js
|
||||||
|
await page.getByRole('button', { name: 'Button 1' }).click();
|
||||||
|
\`\`\`
|
||||||
|
- Snapshot: 002.snapshot.yml
|
||||||
|
|
||||||
`);
|
`);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('session log should record tool user actions', async ({ cdpServer, startClient }, testInfo) => {
|
test('session log should update user action', async ({ cdpServer, startClient }, testInfo) => {
|
||||||
|
const browserContext = await cdpServer.start();
|
||||||
|
const { client, stderr } = await startClient({
|
||||||
|
args: [
|
||||||
|
'--save-session',
|
||||||
|
'--output-dir', testInfo.outputPath('output'),
|
||||||
|
`--cdp-endpoint=${cdpServer.endpoint}`,
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
// Force browser context creation.
|
||||||
|
await client.callTool({
|
||||||
|
name: 'browser_snapshot',
|
||||||
|
});
|
||||||
|
|
||||||
|
const [page] = browserContext.pages();
|
||||||
|
await page.setContent(`
|
||||||
|
<button>Button 1</button>
|
||||||
|
<button>Button 2</button>
|
||||||
|
`);
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: 'Button 1' }).dblclick();
|
||||||
|
|
||||||
|
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||||
|
const sessionFolder = output.substring('Session: '.length);
|
||||||
|
|
||||||
|
await expect.poll(() => readSessionLog(sessionFolder)).toBe(`
|
||||||
|
### Tool call: browser_snapshot
|
||||||
|
- Args
|
||||||
|
\`\`\`json
|
||||||
|
{}
|
||||||
|
\`\`\`
|
||||||
|
- Snapshot: 001.snapshot.yml
|
||||||
|
|
||||||
|
|
||||||
|
### User action: click
|
||||||
|
- Args
|
||||||
|
\`\`\`json
|
||||||
|
{
|
||||||
|
"name": "click",
|
||||||
|
"ref": "e2",
|
||||||
|
"button": "left",
|
||||||
|
"modifiers": 0,
|
||||||
|
"clickCount": 2
|
||||||
|
}
|
||||||
|
\`\`\`
|
||||||
|
- Code
|
||||||
|
\`\`\`js
|
||||||
|
await page.getByRole('button', { name: 'Button 1' }).dblclick();
|
||||||
|
\`\`\`
|
||||||
|
- Snapshot: 002.snapshot.yml
|
||||||
|
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('session log should record tool calls and user actions', async ({ cdpServer, startClient }, testInfo) => {
|
||||||
const browserContext = await cdpServer.start();
|
const browserContext = await cdpServer.start();
|
||||||
const { client, stderr } = await startClient({
|
const { client, stderr } = await startClient({
|
||||||
args: [
|
args: [
|
||||||
@@ -117,8 +226,8 @@ test('session log should record tool user actions', async ({ cdpServer, startCli
|
|||||||
|
|
||||||
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||||
const sessionFolder = output.substring('Session: '.length);
|
const sessionFolder = output.substring('Session: '.length);
|
||||||
const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
|
await expect.poll(() => readSessionLog(sessionFolder)).toBe(`
|
||||||
expect(sessionLog).toBe(`### Tool call: browser_snapshot
|
### Tool call: browser_snapshot
|
||||||
- Args
|
- Args
|
||||||
\`\`\`json
|
\`\`\`json
|
||||||
{}
|
{}
|
||||||
@@ -127,6 +236,16 @@ test('session log should record tool user actions', async ({ cdpServer, startCli
|
|||||||
|
|
||||||
|
|
||||||
### User action: click
|
### User action: click
|
||||||
|
- Args
|
||||||
|
\`\`\`json
|
||||||
|
{
|
||||||
|
"name": "click",
|
||||||
|
"ref": "e2",
|
||||||
|
"button": "left",
|
||||||
|
"modifiers": 0,
|
||||||
|
"clickCount": 1
|
||||||
|
}
|
||||||
|
\`\`\`
|
||||||
- Code
|
- Code
|
||||||
\`\`\`js
|
\`\`\`js
|
||||||
await page.getByRole('button', { name: 'Button 1' }).click();
|
await page.getByRole('button', { name: 'Button 1' }).click();
|
||||||
@@ -148,6 +267,9 @@ await page.getByRole('button', { name: 'Button 2' }).click();
|
|||||||
\`\`\`
|
\`\`\`
|
||||||
- Snapshot: 003.snapshot.yml
|
- Snapshot: 003.snapshot.yml
|
||||||
|
|
||||||
|
|
||||||
`);
|
`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
async function readSessionLog(sessionFolder: string): Promise<string> {
|
||||||
|
return await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8').catch(() => '');
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user