From f60023489799adb72d26df83b24110472fad1196 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 30 Jul 2025 18:26:13 -0700 Subject: [PATCH] chore: record user actions in the session log (#798) --- package-lock.json | 28 +++--- package.json | 6 +- src/actions.d.ts | 172 ++++++++++++++++++++++++++++++++++++ src/browserServerBackend.ts | 29 ++++-- src/context.ts | 100 ++++++++++++++++++++- src/response.ts | 6 +- src/sessionLog.ts | 55 ++++++++++-- src/tab.ts | 7 ++ src/tools/files.ts | 1 - src/tools/tool.ts | 4 +- tests/fixtures.ts | 4 +- tests/screenshot.spec.ts | 25 +++--- tests/session-log.spec.ts | 153 ++++++++++++++++++++++++++++++++ 13 files changed, 536 insertions(+), 54 deletions(-) create mode 100644 src/actions.d.ts create mode 100644 tests/session-log.spec.ts diff --git a/package-lock.json b/package-lock.json index b47a0a5..56ce2f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,8 @@ "debug": "^4.4.1", "dotenv": "^17.2.0", "mime": "^4.0.7", - "playwright": "1.55.0-alpha-1752701791000", - "playwright-core": "1.55.0-alpha-1752701791000", + "playwright": "1.55.0-alpha-1753913825000", + "playwright-core": "1.55.0-alpha-1753913825000", "ws": "^8.18.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.4" @@ -27,7 +27,7 @@ "@anthropic-ai/sdk": "^0.57.0", "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.19.0", - "@playwright/test": "1.55.0-alpha-1752701791000", + "@playwright/test": "1.55.0-alpha-1753913825000", "@stylistic/eslint-plugin": "^3.0.1", "@types/debug": "^4.1.12", "@types/node": "^22.13.10", @@ -703,13 +703,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.55.0-alpha-1752701791000", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.0-alpha-1752701791000.tgz", - "integrity": "sha512-mnitdsjXKPyKTjQQDJ78Or1xZSGcaoDzZVD/0BWFCvygn3nyNmGmiias/Mlfvzvgz9UWBbPeZYxU/bd2Lu+OrQ==", + "version": "1.55.0-alpha-1753913825000", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.0-alpha-1753913825000.tgz", + "integrity": "sha512-YM5YHU6nTYNVzXlKvQvtEdXzpubLvdfEiTxwWvbqGHL/iDK2kBJd3L0psIG6yClU1wy01O756TkOOQSEpzOu7g==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.55.0-alpha-1752701791000" + "playwright": "1.55.0-alpha-1753913825000" }, "bin": { "playwright": "cli.js" @@ -3745,12 +3745,12 @@ } }, "node_modules/playwright": { - "version": "1.55.0-alpha-1752701791000", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0-alpha-1752701791000.tgz", - "integrity": "sha512-PA3TvDz7uQ+Pde0uaii5/WpU5vntRJsYFsaSPoBzywIqzYFO1ugk1ZZ0q6z4/xHq0ha1UClvsv3P77B+u1fi+w==", + "version": "1.55.0-alpha-1753913825000", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0-alpha-1753913825000.tgz", + "integrity": "sha512-IDyZzTu3tRNIjcx7/6ZmU7VmZPFGaW4jNsizwqbjSoeLFZPTLx2y693qeVVF/8KwEjuiSU3hVTQEzWvnx7cf2Q==", "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.55.0-alpha-1752701791000" + "playwright-core": "1.55.0-alpha-1753913825000" }, "bin": { "playwright": "cli.js" @@ -3763,9 +3763,9 @@ } }, "node_modules/playwright-core": { - "version": "1.55.0-alpha-1752701791000", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0-alpha-1752701791000.tgz", - "integrity": "sha512-mQhzhjJMiqnGNnYZv7M4yk1OcNTt1E72jrTLO7EqZuoeat4+qpcU0/mbK+RcTEass5a9YheoVFh6OIhruFMGVg==", + "version": "1.55.0-alpha-1753913825000", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0-alpha-1753913825000.tgz", + "integrity": "sha512-FH5pHzLseQxD8+d2wGlRa/I32AzJ+ZzcdDNM1aiSw5+gmq+aOo3PBqXHvhsh7tj0h4l2Qf6z9qf4mMiwijVthw==", "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" diff --git a/package.json b/package.json index 1357c43..e973292 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "debug": "^4.4.1", "dotenv": "^17.2.0", "mime": "^4.0.7", - "playwright": "1.55.0-alpha-1752701791000", - "playwright-core": "1.55.0-alpha-1752701791000", + "playwright": "1.55.0-alpha-1753913825000", + "playwright-core": "1.55.0-alpha-1753913825000", "ws": "^8.18.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.4" @@ -52,7 +52,7 @@ "@anthropic-ai/sdk": "^0.57.0", "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.19.0", - "@playwright/test": "1.55.0-alpha-1752701791000", + "@playwright/test": "1.55.0-alpha-1753913825000", "@stylistic/eslint-plugin": "^3.0.1", "@types/debug": "^4.1.12", "@types/node": "^22.13.10", diff --git a/src/actions.d.ts b/src/actions.d.ts new file mode 100644 index 0000000..e52b420 --- /dev/null +++ b/src/actions.d.ts @@ -0,0 +1,172 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type Point = { x: number, y: number }; + +export type ActionName = + 'check' | + 'click' | + 'closePage' | + 'fill' | + 'navigate' | + 'openPage' | + 'press' | + 'select' | + 'uncheck' | + 'setInputFiles' | + 'assertText' | + 'assertValue' | + 'assertChecked' | + 'assertVisible' | + 'assertSnapshot'; + +export type ActionBase = { + name: ActionName, + signals: Signal[], + ariaSnapshot?: string, +}; + +export type ActionWithSelector = ActionBase & { + selector: string, + ref?: string, +}; + +export type ClickAction = ActionWithSelector & { + name: 'click', + button: 'left' | 'middle' | 'right', + modifiers: number, + clickCount: number, + position?: Point, +}; + +export type CheckAction = ActionWithSelector & { + name: 'check', +}; + +export type UncheckAction = ActionWithSelector & { + name: 'uncheck', +}; + +export type FillAction = ActionWithSelector & { + name: 'fill', + text: string, +}; + +export type NavigateAction = ActionBase & { + name: 'navigate', + url: string, +}; + +export type OpenPageAction = ActionBase & { + name: 'openPage', + url: string, +}; + +export type ClosesPageAction = ActionBase & { + name: 'closePage', +}; + +export type PressAction = ActionWithSelector & { + name: 'press', + key: string, + modifiers: number, +}; + +export type SelectAction = ActionWithSelector & { + name: 'select', + options: string[], +}; + +export type SetInputFilesAction = ActionWithSelector & { + name: 'setInputFiles', + files: string[], +}; + +export type AssertTextAction = ActionWithSelector & { + name: 'assertText', + text: string, + substring: boolean, +}; + +export type AssertValueAction = ActionWithSelector & { + name: 'assertValue', + value: string, +}; + +export type AssertCheckedAction = ActionWithSelector & { + name: 'assertChecked', + checked: boolean, +}; + +export type AssertVisibleAction = ActionWithSelector & { + name: 'assertVisible', +}; + +export type AssertSnapshotAction = ActionWithSelector & { + name: 'assertSnapshot', + ariaSnapshot: string, +}; + +export type Action = ClickAction | CheckAction | ClosesPageAction | OpenPageAction | UncheckAction | FillAction | NavigateAction | PressAction | SelectAction | SetInputFilesAction | AssertTextAction | AssertValueAction | AssertCheckedAction | AssertVisibleAction | AssertSnapshotAction; +export type AssertAction = AssertCheckedAction | AssertValueAction | AssertTextAction | AssertVisibleAction | AssertSnapshotAction; +export type PerformOnRecordAction = ClickAction | CheckAction | UncheckAction | PressAction | SelectAction; + +// Signals. + +export type BaseSignal = { +}; + +export type NavigationSignal = BaseSignal & { + name: 'navigation', + url: string, +}; + +export type PopupSignal = BaseSignal & { + name: 'popup', + popupAlias: string, +}; + +export type DownloadSignal = BaseSignal & { + name: 'download', + downloadAlias: string, +}; + +export type DialogSignal = BaseSignal & { + name: 'dialog', + dialogAlias: string, +}; + +export type Signal = NavigationSignal | PopupSignal | DownloadSignal | DialogSignal; + +export type FrameDescription = { + pageGuid: string; + pageAlias: string; + framePath: string[]; +}; + +export type ActionInContext = { + frame: FrameDescription; + description?: string; + action: Action; + startTime: number; + endTime?: number; +}; + +export type SignalInContext = { + frame: FrameDescription; + signal: Signal; + timestamp: number; +}; diff --git a/src/browserServerBackend.ts b/src/browserServerBackend.ts index 2c5c54c..75f1fc6 100644 --- a/src/browserServerBackend.ts +++ b/src/browserServerBackend.ts @@ -33,16 +33,20 @@ export class BrowserServerBackend implements ServerBackend { onclose?: () => void; private _tools: Tool[]; - private _context: Context; + private _context: Context | undefined; private _sessionLog: SessionLog | undefined; + private _config: FullConfig; + private _browserContextFactory: BrowserContextFactory; constructor(config: FullConfig, browserContextFactory: BrowserContextFactory) { + this._config = config; + this._browserContextFactory = browserContextFactory; this._tools = filteredTools(config); - this._context = new Context(this._tools, config, browserContextFactory); } async initialize() { - this._sessionLog = this._context.config.saveSession ? await SessionLog.create(this._context.config) : undefined; + this._sessionLog = this._config.saveSession ? await SessionLog.create(this._config) : undefined; + this._context = new Context(this._tools, this._config, this._browserContextFactory, this._sessionLog); } tools(): mcpServer.ToolSchema[] { @@ -50,20 +54,27 @@ export class BrowserServerBackend implements ServerBackend { } async callTool(schema: mcpServer.ToolSchema, parsedArguments: any) { - const response = new Response(this._context, schema.name, parsedArguments); + const context = this._context!; + const response = new Response(context, schema.name, parsedArguments); const tool = this._tools.find(tool => tool.schema.name === schema.name)!; - await tool.handle(this._context, parsedArguments, response); - if (this._sessionLog) - await this._sessionLog.log(response); + await context.setInputRecorderEnabled(false); + try { + await tool.handle(context, parsedArguments, response); + } catch (error) { + response.addError(String(error)); + } finally { + await context.setInputRecorderEnabled(true); + } + await this._sessionLog?.logResponse(response); return await response.serialize(); } serverInitialized(version: mcpServer.ClientVersion | undefined) { - this._context.clientVersion = version; + this._context!.clientVersion = version; } serverClosed() { this.onclose?.(); - void this._context.dispose().catch(logUnhandledError); + void this._context!.dispose().catch(logUnhandledError); } } diff --git a/src/context.ts b/src/context.ts index 42c9c3f..46aa5d4 100644 --- a/src/context.ts +++ b/src/context.ts @@ -23,6 +23,8 @@ import { Tab } from './tab.js'; import type { Tool } from './tools/tool.js'; import type { FullConfig } from './config.js'; import type { BrowserContextFactory } from './browserContextFactory.js'; +import type * as actions from './actions.js'; +import type { Action, SessionLog } from './sessionLog.js'; const testDebug = debug('pw:mcp:test'); @@ -33,15 +35,19 @@ export class Context { private _browserContextFactory: BrowserContextFactory; private _tabs: Tab[] = []; private _currentTab: Tab | undefined; + clientVersion: { name: string; version: string; } | undefined; private static _allContexts: Set = new Set(); private _closeBrowserContextPromise: Promise | undefined; + private _inputRecorder: InputRecorder | undefined; + private _sessionLog: SessionLog | undefined; - constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory) { + constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory, sessionLog: SessionLog | undefined) { this.tools = tools; this.config = config; this._browserContextFactory = browserContextFactory; + this._sessionLog = sessionLog; testDebug('create context'); Context._allContexts.add(this); } @@ -146,6 +152,10 @@ export class Context { this._closeBrowserContextPromise = undefined; } + async setInputRecorderEnabled(enabled: boolean) { + await this._inputRecorder?.setEnabled(enabled); + } + private async _closeBrowserContextImpl() { if (!this._browserContextPromise) return; @@ -198,6 +208,8 @@ export class Context { const result = await this._browserContextFactory.createContext(this.clientVersion!); const { browserContext } = result; await this._setupRequestInterception(browserContext); + if (this._sessionLog) + this._inputRecorder = await InputRecorder.create(this._sessionLog, browserContext); for (const page of browserContext.pages()) this._onPageCreated(page); browserContext.on('page', page => this._onPageCreated(page)); @@ -212,3 +224,89 @@ export class Context { return result; } } + +export class InputRecorder { + private _actions: Action[] = []; + private _enabled = false; + private _sessionLog: SessionLog; + private _browserContext: playwright.BrowserContext; + private _flushTimer: NodeJS.Timeout | undefined; + + private constructor(sessionLog: SessionLog, browserContext: playwright.BrowserContext) { + this._sessionLog = sessionLog; + this._browserContext = browserContext; + } + + static async create(sessionLog: SessionLog, browserContext: playwright.BrowserContext) { + const recorder = new InputRecorder(sessionLog, browserContext); + await recorder._initialize(); + await recorder.setEnabled(true); + return recorder; + } + + private async _initialize() { + await (this._browserContext as any)._enableRecorder({ + mode: 'recording', + recorderMode: 'api', + }, { + actionAdded: (page: playwright.Page, data: actions.ActionInContext, code: string) => { + if (!this._enabled) + return; + const tab = Tab.forPage(page); + this._actions.push({ ...data, tab, code: code.trim(), timestamp: performance.now() }); + this._scheduleFlush(); + }, + actionUpdated: (page: playwright.Page, data: actions.ActionInContext, code: string) => { + if (!this._enabled) + return; + const tab = Tab.forPage(page); + this._actions[this._actions.length - 1] = { ...data, tab, code: code.trim(), timestamp: performance.now() }; + this._scheduleFlush(); + }, + signalAdded: (page: playwright.Page, data: actions.SignalInContext) => { + if (data.signal.name !== 'navigation') + return; + const tab = Tab.forPage(page); + this._actions.push({ + frame: data.frame, + action: { + name: 'navigate', + url: data.signal.url, + signals: [], + }, + startTime: data.timestamp, + endTime: data.timestamp, + tab, + code: `await page.goto('${data.signal.url}');`, + timestamp: performance.now(), + }); + this._scheduleFlush(); + }, + }); + } + + async setEnabled(enabled: boolean) { + this._enabled = enabled; + if (!enabled) + await this._flush(); + } + + private _clearTimer() { + if (this._flushTimer) { + clearTimeout(this._flushTimer); + this._flushTimer = undefined; + } + } + + private _scheduleFlush() { + this._clearTimer(); + this._flushTimer = setTimeout(() => this._flush(), 1000); + } + + private async _flush() { + this._clearTimer(); + const actions = this._actions; + this._actions = []; + await this._sessionLog.logActions(actions); + } +} diff --git a/src/response.ts b/src/response.ts index 8cd0c8f..98d77b6 100644 --- a/src/response.ts +++ b/src/response.ts @@ -41,10 +41,14 @@ export class Response { } addError(error: string) { - this._result.push(`Error: ${error}`); + this._result.push(error); this._isError = true; } + isError() { + return this._isError; + } + result() { return this._result.join('\n'); } diff --git a/src/sessionLog.ts b/src/sessionLog.ts index 0dedab8..21a0053 100644 --- a/src/sessionLog.ts +++ b/src/sessionLog.ts @@ -20,13 +20,16 @@ import path from 'path'; import { outputFile } from './config.js'; import { Response } from './response.js'; import type { FullConfig } from './config.js'; +import type * as actions from './actions.js'; +import type { Tab } from './tab.js'; -let sessionOrdinal = 0; +export type Action = actions.ActionInContext & { code: string; tab?: Tab | undefined; timestamp: number }; export class SessionLog { private _folder: string; private _file: string; private _ordinal = 0; + private _lastModified = 0; constructor(sessionFolder: string) { this._folder = sessionFolder; @@ -34,18 +37,22 @@ export class SessionLog { } static async create(config: FullConfig): Promise { - const sessionFolder = await outputFile(config, `session-${(++sessionOrdinal).toString().padStart(3, '0')}`); + const sessionFolder = await outputFile(config, `session-${Date.now()}`); await fs.promises.mkdir(sessionFolder, { recursive: true }); // eslint-disable-next-line no-console console.error(`Session: ${sessionFolder}`); return new SessionLog(sessionFolder); } - async log(response: Response) { + lastModified() { + return this._lastModified; + } + + async logResponse(response: Response) { + this._lastModified = performance.now(); const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`; const lines: string[] = [ - `### Tool: ${response.toolName}`, - ``, + `### Tool call: ${response.toolName}`, `- Args`, '```json', JSON.stringify(response.toolArgs, null, 2), @@ -53,7 +60,7 @@ export class SessionLog { ]; if (response.result()) { lines.push( - `- Result`, + response.isError() ? `- Error` : `- Result`, '```', response.result(), '```'); @@ -80,7 +87,41 @@ export class SessionLog { lines.push(`- Screenshot: ${fileName}`); } - lines.push('', ''); + lines.push('', '', ''); + await this._appendLines(lines); + } + + async logActions(actions: Action[]) { + // Skip recent navigation, it is a side-effect of the previous action or tool use. + if (actions?.[0]?.action?.name === 'navigate' && actions[0].timestamp - this._lastModified < 1000) + return; + + this._lastModified = performance.now(); + const lines: string[] = []; + for (const action of actions) { + const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`; + lines.push( + `### User action: ${action.action.name}`, + ); + if (action.code) { + lines.push( + `- Code`, + '```js', + action.code, + '```'); + } + if (action.action.ariaSnapshot) { + const fileName = `${prefix}.snapshot.yml`; + await fs.promises.writeFile(path.join(this._folder, fileName), action.action.ariaSnapshot); + lines.push(`- Snapshot: ${fileName}`); + } + lines.push('', '', ''); + } + + await this._appendLines(lines); + } + + private async _appendLines(lines: string[]) { await fs.promises.appendFile(this._file, lines.join('\n')); } } diff --git a/src/tab.ts b/src/tab.ts index ca3e101..23ca659 100644 --- a/src/tab.ts +++ b/src/tab.ts @@ -69,6 +69,11 @@ export class Tab extends EventEmitter { }); page.setDefaultNavigationTimeout(60000); page.setDefaultTimeout(5000); + (page as any)[tabSymbol] = this; + } + + static forPage(page: playwright.Page): Tab | undefined { + return (page as any)[tabSymbol]; } modalStates(): ModalState[] { @@ -308,3 +313,5 @@ function trim(text: string, maxLength: number) { return text; return text.slice(0, maxLength) + '...'; } + +const tabSymbol = Symbol('tabSymbol'); diff --git a/src/tools/files.ts b/src/tools/files.ts index 939aa3c..2097b7b 100644 --- a/src/tools/files.ts +++ b/src/tools/files.ts @@ -37,7 +37,6 @@ const uploadFile = defineTabTool({ if (!modalState) throw new Error('No file chooser visible'); - response.addCode(`// Select files for upload`); response.addCode(`await fileChooser.setFiles(${JSON.stringify(params.paths)})`); tab.clearModalState(modalState); diff --git a/src/tools/tool.ts b/src/tools/tool.ts index 4733507..f0b7795 100644 --- a/src/tools/tool.ts +++ b/src/tools/tool.ts @@ -60,9 +60,9 @@ export function defineTabTool(tool: TabTool): Too const tab = context.currentTabOrDie(); const modalStates = tab.modalStates().map(state => state.type); if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState)) - response.addError(`The tool "${tool.schema.name}" can only be used when there is related modal state present.\n` + tab.modalStatesMarkdown().join('\n')); + response.addError(`Error: The tool "${tool.schema.name}" can only be used when there is related modal state present.\n` + tab.modalStatesMarkdown().join('\n')); else if (!tool.clearsModalState && modalStates.length) - response.addError(`Tool "${tool.schema.name}" does not handle the modal state.\n` + tab.modalStatesMarkdown().join('\n')); + response.addError(`Error: Tool "${tool.schema.name}" does not handle the modal state.\n` + tab.modalStatesMarkdown().join('\n')); else return tool.handle(tab, params, response); }, diff --git a/tests/fixtures.ts b/tests/fixtures.ts index 783c91e..b262ad5 100644 --- a/tests/fixtures.ts +++ b/tests/fixtures.ts @@ -225,7 +225,7 @@ export function formatOutput(output: string): string[] { } function parseResponse(response: any) { - const text = (response as any).content[0].text; + const text = response.content[0].text; const sections = parseSections(text); const result = sections.get('Result'); @@ -237,6 +237,7 @@ function parseResponse(response: any) { const downloads = sections.get('Downloads'); const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, ''); const isError = response.isError; + const attachments = response.content.slice(1); return { result, @@ -247,6 +248,7 @@ function parseResponse(response: any) { modalState, downloads, isError, + attachments, }; } diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index f1daa3b..f062f1c 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -31,18 +31,13 @@ test('browser_take_screenshot (viewport)', async ({ startClient, server }, testI expect(await client.callTool({ name: 'browser_take_screenshot', - })).toEqual({ - content: [ - { - text: expect.stringContaining(`Screenshot viewport and save it as`), - type: 'text', - }, - { - data: expect.any(String), - mimeType: 'image/jpeg', - type: 'image', - }, - ], + })).toHaveResponse({ + code: expect.stringContaining(`await page.screenshot`), + attachments: [{ + data: expect.any(String), + mimeType: 'image/jpeg', + type: 'image', + }], }); }); @@ -207,7 +202,7 @@ test('browser_take_screenshot (imageResponses=omit)', async ({ startClient, serv })).toEqual({ content: [ { - text: expect.stringContaining(`Screenshot viewport and save it as`), + text: expect.stringContaining(`await page.screenshot`), type: 'text', }, ], @@ -231,7 +226,7 @@ test('browser_take_screenshot (fullPage: true)', async ({ startClient, server }, })).toEqual({ content: [ { - text: expect.stringContaining(`Screenshot full page and save it as`), + text: expect.stringContaining('fullPage: true'), type: 'text', }, { @@ -285,7 +280,7 @@ test('browser_take_screenshot (viewport without snapshot)', async ({ startClient })).toEqual({ content: [ { - text: expect.stringContaining(`Screenshot viewport and save it as`), + text: expect.stringContaining(`page.screenshot`), type: 'text', }, { diff --git a/tests/session-log.spec.ts b/tests/session-log.spec.ts new file mode 100644 index 0000000..dccc6a9 --- /dev/null +++ b/tests/session-log.spec.ts @@ -0,0 +1,153 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import { test, expect } from './fixtures.js'; + +test('session log should record tool calls', async ({ startClient, server }, testInfo) => { + const { client, stderr } = await startClient({ + args: [ + '--save-session', + '--output-dir', testInfo.outputPath('output'), + ], + }); + + server.setContent('/', `Title`, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_click', + arguments: { + element: 'Submit button', + ref: 'e2', + }, + })).toHaveResponse({ + code: `await page.getByRole('button', { name: 'Submit' }).click();`, + pageState: expect.stringContaining(`- button "Submit"`), + }); + + const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0]; + const sessionFolder = output.substring('Session: '.length); + const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8'); + expect(sessionLog).toBe(`### Tool call: browser_navigate +- Args +\`\`\`json +{ + "url": "http://localhost:${server.PORT}/" +} +\`\`\` +- Code +\`\`\`js +await page.goto('http://localhost:${server.PORT}/'); +\`\`\` +- Snapshot: 001.snapshot.yml + + +### Tool call: browser_click +- Args +\`\`\`json +{ + "element": "Submit button", + "ref": "e2" +} +\`\`\` +- Code +\`\`\`js +await page.getByRole('button', { name: 'Submit' }).click(); +\`\`\` +- Snapshot: 002.snapshot.yml + + +`); +}); + +test('session log should record tool user actions', async ({ cdpServer, startClient }, testInfo) => { + const browserContext = await cdpServer.start(); + const { client, stderr } = await startClient({ + args: [ + '--save-session', + '--output-dir', testInfo.outputPath('output'), + `--cdp-endpoint=${cdpServer.endpoint}`, + ], + }); + + const [page] = browserContext.pages(); + await page.setContent(` + + + `); + + await client.callTool({ + name: 'browser_snapshot', + }); + + // Manual action. + await page.getByRole('button', { name: 'Button 1' }).click(); + + // This is to simulate a delay after the user action before the tool action. + await new Promise(resolve => setTimeout(resolve, 1000)); + + // Tool action. + await client.callTool({ + name: 'browser_click', + arguments: { + element: 'Button 2', + ref: 'e3', + }, + }); + + const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0]; + const sessionFolder = output.substring('Session: '.length); + const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8'); + expect(sessionLog).toBe(`### Tool call: browser_snapshot +- Args +\`\`\`json +{} +\`\`\` +- Snapshot: 001.snapshot.yml + + +### User action: click +- Code +\`\`\`js +await page.getByRole('button', { name: 'Button 1' }).click(); +\`\`\` +- Snapshot: 002.snapshot.yml + + +### Tool call: browser_click +- Args +\`\`\`json +{ + "element": "Button 2", + "ref": "e3" +} +\`\`\` +- Code +\`\`\`js +await page.getByRole('button', { name: 'Button 2' }).click(); +\`\`\` +- Snapshot: 003.snapshot.yml + + +`); +});