chore: record user actions in the session log (#798)
This commit is contained in:
28
package-lock.json
generated
28
package-lock.json
generated
@@ -14,8 +14,8 @@
|
||||
"debug": "^4.4.1",
|
||||
"dotenv": "^17.2.0",
|
||||
"mime": "^4.0.7",
|
||||
"playwright": "1.55.0-alpha-1752701791000",
|
||||
"playwright-core": "1.55.0-alpha-1752701791000",
|
||||
"playwright": "1.55.0-alpha-1753913825000",
|
||||
"playwright-core": "1.55.0-alpha-1753913825000",
|
||||
"ws": "^8.18.1",
|
||||
"zod": "^3.24.1",
|
||||
"zod-to-json-schema": "^3.24.4"
|
||||
@@ -27,7 +27,7 @@
|
||||
"@anthropic-ai/sdk": "^0.57.0",
|
||||
"@eslint/eslintrc": "^3.2.0",
|
||||
"@eslint/js": "^9.19.0",
|
||||
"@playwright/test": "1.55.0-alpha-1752701791000",
|
||||
"@playwright/test": "1.55.0-alpha-1753913825000",
|
||||
"@stylistic/eslint-plugin": "^3.0.1",
|
||||
"@types/debug": "^4.1.12",
|
||||
"@types/node": "^22.13.10",
|
||||
@@ -703,13 +703,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.55.0-alpha-1752701791000",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.0-alpha-1752701791000.tgz",
|
||||
"integrity": "sha512-mnitdsjXKPyKTjQQDJ78Or1xZSGcaoDzZVD/0BWFCvygn3nyNmGmiias/Mlfvzvgz9UWBbPeZYxU/bd2Lu+OrQ==",
|
||||
"version": "1.55.0-alpha-1753913825000",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.0-alpha-1753913825000.tgz",
|
||||
"integrity": "sha512-YM5YHU6nTYNVzXlKvQvtEdXzpubLvdfEiTxwWvbqGHL/iDK2kBJd3L0psIG6yClU1wy01O756TkOOQSEpzOu7g==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.55.0-alpha-1752701791000"
|
||||
"playwright": "1.55.0-alpha-1753913825000"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
@@ -3745,12 +3745,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.55.0-alpha-1752701791000",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0-alpha-1752701791000.tgz",
|
||||
"integrity": "sha512-PA3TvDz7uQ+Pde0uaii5/WpU5vntRJsYFsaSPoBzywIqzYFO1ugk1ZZ0q6z4/xHq0ha1UClvsv3P77B+u1fi+w==",
|
||||
"version": "1.55.0-alpha-1753913825000",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0-alpha-1753913825000.tgz",
|
||||
"integrity": "sha512-IDyZzTu3tRNIjcx7/6ZmU7VmZPFGaW4jNsizwqbjSoeLFZPTLx2y693qeVVF/8KwEjuiSU3hVTQEzWvnx7cf2Q==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.55.0-alpha-1752701791000"
|
||||
"playwright-core": "1.55.0-alpha-1753913825000"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
@@ -3763,9 +3763,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.55.0-alpha-1752701791000",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0-alpha-1752701791000.tgz",
|
||||
"integrity": "sha512-mQhzhjJMiqnGNnYZv7M4yk1OcNTt1E72jrTLO7EqZuoeat4+qpcU0/mbK+RcTEass5a9YheoVFh6OIhruFMGVg==",
|
||||
"version": "1.55.0-alpha-1753913825000",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0-alpha-1753913825000.tgz",
|
||||
"integrity": "sha512-FH5pHzLseQxD8+d2wGlRa/I32AzJ+ZzcdDNM1aiSw5+gmq+aOo3PBqXHvhsh7tj0h4l2Qf6z9qf4mMiwijVthw==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
|
||||
@@ -42,8 +42,8 @@
|
||||
"debug": "^4.4.1",
|
||||
"dotenv": "^17.2.0",
|
||||
"mime": "^4.0.7",
|
||||
"playwright": "1.55.0-alpha-1752701791000",
|
||||
"playwright-core": "1.55.0-alpha-1752701791000",
|
||||
"playwright": "1.55.0-alpha-1753913825000",
|
||||
"playwright-core": "1.55.0-alpha-1753913825000",
|
||||
"ws": "^8.18.1",
|
||||
"zod": "^3.24.1",
|
||||
"zod-to-json-schema": "^3.24.4"
|
||||
@@ -52,7 +52,7 @@
|
||||
"@anthropic-ai/sdk": "^0.57.0",
|
||||
"@eslint/eslintrc": "^3.2.0",
|
||||
"@eslint/js": "^9.19.0",
|
||||
"@playwright/test": "1.55.0-alpha-1752701791000",
|
||||
"@playwright/test": "1.55.0-alpha-1753913825000",
|
||||
"@stylistic/eslint-plugin": "^3.0.1",
|
||||
"@types/debug": "^4.1.12",
|
||||
"@types/node": "^22.13.10",
|
||||
|
||||
172
src/actions.d.ts
vendored
Normal file
172
src/actions.d.ts
vendored
Normal file
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
type Point = { x: number, y: number };
|
||||
|
||||
export type ActionName =
|
||||
'check' |
|
||||
'click' |
|
||||
'closePage' |
|
||||
'fill' |
|
||||
'navigate' |
|
||||
'openPage' |
|
||||
'press' |
|
||||
'select' |
|
||||
'uncheck' |
|
||||
'setInputFiles' |
|
||||
'assertText' |
|
||||
'assertValue' |
|
||||
'assertChecked' |
|
||||
'assertVisible' |
|
||||
'assertSnapshot';
|
||||
|
||||
export type ActionBase = {
|
||||
name: ActionName,
|
||||
signals: Signal[],
|
||||
ariaSnapshot?: string,
|
||||
};
|
||||
|
||||
export type ActionWithSelector = ActionBase & {
|
||||
selector: string,
|
||||
ref?: string,
|
||||
};
|
||||
|
||||
export type ClickAction = ActionWithSelector & {
|
||||
name: 'click',
|
||||
button: 'left' | 'middle' | 'right',
|
||||
modifiers: number,
|
||||
clickCount: number,
|
||||
position?: Point,
|
||||
};
|
||||
|
||||
export type CheckAction = ActionWithSelector & {
|
||||
name: 'check',
|
||||
};
|
||||
|
||||
export type UncheckAction = ActionWithSelector & {
|
||||
name: 'uncheck',
|
||||
};
|
||||
|
||||
export type FillAction = ActionWithSelector & {
|
||||
name: 'fill',
|
||||
text: string,
|
||||
};
|
||||
|
||||
export type NavigateAction = ActionBase & {
|
||||
name: 'navigate',
|
||||
url: string,
|
||||
};
|
||||
|
||||
export type OpenPageAction = ActionBase & {
|
||||
name: 'openPage',
|
||||
url: string,
|
||||
};
|
||||
|
||||
export type ClosesPageAction = ActionBase & {
|
||||
name: 'closePage',
|
||||
};
|
||||
|
||||
export type PressAction = ActionWithSelector & {
|
||||
name: 'press',
|
||||
key: string,
|
||||
modifiers: number,
|
||||
};
|
||||
|
||||
export type SelectAction = ActionWithSelector & {
|
||||
name: 'select',
|
||||
options: string[],
|
||||
};
|
||||
|
||||
export type SetInputFilesAction = ActionWithSelector & {
|
||||
name: 'setInputFiles',
|
||||
files: string[],
|
||||
};
|
||||
|
||||
export type AssertTextAction = ActionWithSelector & {
|
||||
name: 'assertText',
|
||||
text: string,
|
||||
substring: boolean,
|
||||
};
|
||||
|
||||
export type AssertValueAction = ActionWithSelector & {
|
||||
name: 'assertValue',
|
||||
value: string,
|
||||
};
|
||||
|
||||
export type AssertCheckedAction = ActionWithSelector & {
|
||||
name: 'assertChecked',
|
||||
checked: boolean,
|
||||
};
|
||||
|
||||
export type AssertVisibleAction = ActionWithSelector & {
|
||||
name: 'assertVisible',
|
||||
};
|
||||
|
||||
export type AssertSnapshotAction = ActionWithSelector & {
|
||||
name: 'assertSnapshot',
|
||||
ariaSnapshot: string,
|
||||
};
|
||||
|
||||
export type Action = ClickAction | CheckAction | ClosesPageAction | OpenPageAction | UncheckAction | FillAction | NavigateAction | PressAction | SelectAction | SetInputFilesAction | AssertTextAction | AssertValueAction | AssertCheckedAction | AssertVisibleAction | AssertSnapshotAction;
|
||||
export type AssertAction = AssertCheckedAction | AssertValueAction | AssertTextAction | AssertVisibleAction | AssertSnapshotAction;
|
||||
export type PerformOnRecordAction = ClickAction | CheckAction | UncheckAction | PressAction | SelectAction;
|
||||
|
||||
// Signals.
|
||||
|
||||
export type BaseSignal = {
|
||||
};
|
||||
|
||||
export type NavigationSignal = BaseSignal & {
|
||||
name: 'navigation',
|
||||
url: string,
|
||||
};
|
||||
|
||||
export type PopupSignal = BaseSignal & {
|
||||
name: 'popup',
|
||||
popupAlias: string,
|
||||
};
|
||||
|
||||
export type DownloadSignal = BaseSignal & {
|
||||
name: 'download',
|
||||
downloadAlias: string,
|
||||
};
|
||||
|
||||
export type DialogSignal = BaseSignal & {
|
||||
name: 'dialog',
|
||||
dialogAlias: string,
|
||||
};
|
||||
|
||||
export type Signal = NavigationSignal | PopupSignal | DownloadSignal | DialogSignal;
|
||||
|
||||
export type FrameDescription = {
|
||||
pageGuid: string;
|
||||
pageAlias: string;
|
||||
framePath: string[];
|
||||
};
|
||||
|
||||
export type ActionInContext = {
|
||||
frame: FrameDescription;
|
||||
description?: string;
|
||||
action: Action;
|
||||
startTime: number;
|
||||
endTime?: number;
|
||||
};
|
||||
|
||||
export type SignalInContext = {
|
||||
frame: FrameDescription;
|
||||
signal: Signal;
|
||||
timestamp: number;
|
||||
};
|
||||
@@ -33,16 +33,20 @@ export class BrowserServerBackend implements ServerBackend {
|
||||
onclose?: () => void;
|
||||
|
||||
private _tools: Tool[];
|
||||
private _context: Context;
|
||||
private _context: Context | undefined;
|
||||
private _sessionLog: SessionLog | undefined;
|
||||
private _config: FullConfig;
|
||||
private _browserContextFactory: BrowserContextFactory;
|
||||
|
||||
constructor(config: FullConfig, browserContextFactory: BrowserContextFactory) {
|
||||
this._config = config;
|
||||
this._browserContextFactory = browserContextFactory;
|
||||
this._tools = filteredTools(config);
|
||||
this._context = new Context(this._tools, config, browserContextFactory);
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
this._sessionLog = this._context.config.saveSession ? await SessionLog.create(this._context.config) : undefined;
|
||||
this._sessionLog = this._config.saveSession ? await SessionLog.create(this._config) : undefined;
|
||||
this._context = new Context(this._tools, this._config, this._browserContextFactory, this._sessionLog);
|
||||
}
|
||||
|
||||
tools(): mcpServer.ToolSchema<any>[] {
|
||||
@@ -50,20 +54,27 @@ export class BrowserServerBackend implements ServerBackend {
|
||||
}
|
||||
|
||||
async callTool(schema: mcpServer.ToolSchema<any>, parsedArguments: any) {
|
||||
const response = new Response(this._context, schema.name, parsedArguments);
|
||||
const context = this._context!;
|
||||
const response = new Response(context, schema.name, parsedArguments);
|
||||
const tool = this._tools.find(tool => tool.schema.name === schema.name)!;
|
||||
await tool.handle(this._context, parsedArguments, response);
|
||||
if (this._sessionLog)
|
||||
await this._sessionLog.log(response);
|
||||
await context.setInputRecorderEnabled(false);
|
||||
try {
|
||||
await tool.handle(context, parsedArguments, response);
|
||||
} catch (error) {
|
||||
response.addError(String(error));
|
||||
} finally {
|
||||
await context.setInputRecorderEnabled(true);
|
||||
}
|
||||
await this._sessionLog?.logResponse(response);
|
||||
return await response.serialize();
|
||||
}
|
||||
|
||||
serverInitialized(version: mcpServer.ClientVersion | undefined) {
|
||||
this._context.clientVersion = version;
|
||||
this._context!.clientVersion = version;
|
||||
}
|
||||
|
||||
serverClosed() {
|
||||
this.onclose?.();
|
||||
void this._context.dispose().catch(logUnhandledError);
|
||||
void this._context!.dispose().catch(logUnhandledError);
|
||||
}
|
||||
}
|
||||
|
||||
100
src/context.ts
100
src/context.ts
@@ -23,6 +23,8 @@ import { Tab } from './tab.js';
|
||||
import type { Tool } from './tools/tool.js';
|
||||
import type { FullConfig } from './config.js';
|
||||
import type { BrowserContextFactory } from './browserContextFactory.js';
|
||||
import type * as actions from './actions.js';
|
||||
import type { Action, SessionLog } from './sessionLog.js';
|
||||
|
||||
const testDebug = debug('pw:mcp:test');
|
||||
|
||||
@@ -33,15 +35,19 @@ export class Context {
|
||||
private _browserContextFactory: BrowserContextFactory;
|
||||
private _tabs: Tab[] = [];
|
||||
private _currentTab: Tab | undefined;
|
||||
|
||||
clientVersion: { name: string; version: string; } | undefined;
|
||||
|
||||
private static _allContexts: Set<Context> = new Set();
|
||||
private _closeBrowserContextPromise: Promise<void> | undefined;
|
||||
private _inputRecorder: InputRecorder | undefined;
|
||||
private _sessionLog: SessionLog | undefined;
|
||||
|
||||
constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory) {
|
||||
constructor(tools: Tool[], config: FullConfig, browserContextFactory: BrowserContextFactory, sessionLog: SessionLog | undefined) {
|
||||
this.tools = tools;
|
||||
this.config = config;
|
||||
this._browserContextFactory = browserContextFactory;
|
||||
this._sessionLog = sessionLog;
|
||||
testDebug('create context');
|
||||
Context._allContexts.add(this);
|
||||
}
|
||||
@@ -146,6 +152,10 @@ export class Context {
|
||||
this._closeBrowserContextPromise = undefined;
|
||||
}
|
||||
|
||||
async setInputRecorderEnabled(enabled: boolean) {
|
||||
await this._inputRecorder?.setEnabled(enabled);
|
||||
}
|
||||
|
||||
private async _closeBrowserContextImpl() {
|
||||
if (!this._browserContextPromise)
|
||||
return;
|
||||
@@ -198,6 +208,8 @@ export class Context {
|
||||
const result = await this._browserContextFactory.createContext(this.clientVersion!);
|
||||
const { browserContext } = result;
|
||||
await this._setupRequestInterception(browserContext);
|
||||
if (this._sessionLog)
|
||||
this._inputRecorder = await InputRecorder.create(this._sessionLog, browserContext);
|
||||
for (const page of browserContext.pages())
|
||||
this._onPageCreated(page);
|
||||
browserContext.on('page', page => this._onPageCreated(page));
|
||||
@@ -212,3 +224,89 @@ export class Context {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
export class InputRecorder {
|
||||
private _actions: Action[] = [];
|
||||
private _enabled = false;
|
||||
private _sessionLog: SessionLog;
|
||||
private _browserContext: playwright.BrowserContext;
|
||||
private _flushTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
private constructor(sessionLog: SessionLog, browserContext: playwright.BrowserContext) {
|
||||
this._sessionLog = sessionLog;
|
||||
this._browserContext = browserContext;
|
||||
}
|
||||
|
||||
static async create(sessionLog: SessionLog, browserContext: playwright.BrowserContext) {
|
||||
const recorder = new InputRecorder(sessionLog, browserContext);
|
||||
await recorder._initialize();
|
||||
await recorder.setEnabled(true);
|
||||
return recorder;
|
||||
}
|
||||
|
||||
private async _initialize() {
|
||||
await (this._browserContext as any)._enableRecorder({
|
||||
mode: 'recording',
|
||||
recorderMode: 'api',
|
||||
}, {
|
||||
actionAdded: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
||||
if (!this._enabled)
|
||||
return;
|
||||
const tab = Tab.forPage(page);
|
||||
this._actions.push({ ...data, tab, code: code.trim(), timestamp: performance.now() });
|
||||
this._scheduleFlush();
|
||||
},
|
||||
actionUpdated: (page: playwright.Page, data: actions.ActionInContext, code: string) => {
|
||||
if (!this._enabled)
|
||||
return;
|
||||
const tab = Tab.forPage(page);
|
||||
this._actions[this._actions.length - 1] = { ...data, tab, code: code.trim(), timestamp: performance.now() };
|
||||
this._scheduleFlush();
|
||||
},
|
||||
signalAdded: (page: playwright.Page, data: actions.SignalInContext) => {
|
||||
if (data.signal.name !== 'navigation')
|
||||
return;
|
||||
const tab = Tab.forPage(page);
|
||||
this._actions.push({
|
||||
frame: data.frame,
|
||||
action: {
|
||||
name: 'navigate',
|
||||
url: data.signal.url,
|
||||
signals: [],
|
||||
},
|
||||
startTime: data.timestamp,
|
||||
endTime: data.timestamp,
|
||||
tab,
|
||||
code: `await page.goto('${data.signal.url}');`,
|
||||
timestamp: performance.now(),
|
||||
});
|
||||
this._scheduleFlush();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async setEnabled(enabled: boolean) {
|
||||
this._enabled = enabled;
|
||||
if (!enabled)
|
||||
await this._flush();
|
||||
}
|
||||
|
||||
private _clearTimer() {
|
||||
if (this._flushTimer) {
|
||||
clearTimeout(this._flushTimer);
|
||||
this._flushTimer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private _scheduleFlush() {
|
||||
this._clearTimer();
|
||||
this._flushTimer = setTimeout(() => this._flush(), 1000);
|
||||
}
|
||||
|
||||
private async _flush() {
|
||||
this._clearTimer();
|
||||
const actions = this._actions;
|
||||
this._actions = [];
|
||||
await this._sessionLog.logActions(actions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,10 +41,14 @@ export class Response {
|
||||
}
|
||||
|
||||
addError(error: string) {
|
||||
this._result.push(`Error: ${error}`);
|
||||
this._result.push(error);
|
||||
this._isError = true;
|
||||
}
|
||||
|
||||
isError() {
|
||||
return this._isError;
|
||||
}
|
||||
|
||||
result() {
|
||||
return this._result.join('\n');
|
||||
}
|
||||
|
||||
@@ -20,13 +20,16 @@ import path from 'path';
|
||||
import { outputFile } from './config.js';
|
||||
import { Response } from './response.js';
|
||||
import type { FullConfig } from './config.js';
|
||||
import type * as actions from './actions.js';
|
||||
import type { Tab } from './tab.js';
|
||||
|
||||
let sessionOrdinal = 0;
|
||||
export type Action = actions.ActionInContext & { code: string; tab?: Tab | undefined; timestamp: number };
|
||||
|
||||
export class SessionLog {
|
||||
private _folder: string;
|
||||
private _file: string;
|
||||
private _ordinal = 0;
|
||||
private _lastModified = 0;
|
||||
|
||||
constructor(sessionFolder: string) {
|
||||
this._folder = sessionFolder;
|
||||
@@ -34,18 +37,22 @@ export class SessionLog {
|
||||
}
|
||||
|
||||
static async create(config: FullConfig): Promise<SessionLog> {
|
||||
const sessionFolder = await outputFile(config, `session-${(++sessionOrdinal).toString().padStart(3, '0')}`);
|
||||
const sessionFolder = await outputFile(config, `session-${Date.now()}`);
|
||||
await fs.promises.mkdir(sessionFolder, { recursive: true });
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(`Session: ${sessionFolder}`);
|
||||
return new SessionLog(sessionFolder);
|
||||
}
|
||||
|
||||
async log(response: Response) {
|
||||
lastModified() {
|
||||
return this._lastModified;
|
||||
}
|
||||
|
||||
async logResponse(response: Response) {
|
||||
this._lastModified = performance.now();
|
||||
const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`;
|
||||
const lines: string[] = [
|
||||
`### Tool: ${response.toolName}`,
|
||||
``,
|
||||
`### Tool call: ${response.toolName}`,
|
||||
`- Args`,
|
||||
'```json',
|
||||
JSON.stringify(response.toolArgs, null, 2),
|
||||
@@ -53,7 +60,7 @@ export class SessionLog {
|
||||
];
|
||||
if (response.result()) {
|
||||
lines.push(
|
||||
`- Result`,
|
||||
response.isError() ? `- Error` : `- Result`,
|
||||
'```',
|
||||
response.result(),
|
||||
'```');
|
||||
@@ -80,7 +87,41 @@ export class SessionLog {
|
||||
lines.push(`- Screenshot: ${fileName}`);
|
||||
}
|
||||
|
||||
lines.push('', '');
|
||||
lines.push('', '', '');
|
||||
await this._appendLines(lines);
|
||||
}
|
||||
|
||||
async logActions(actions: Action[]) {
|
||||
// Skip recent navigation, it is a side-effect of the previous action or tool use.
|
||||
if (actions?.[0]?.action?.name === 'navigate' && actions[0].timestamp - this._lastModified < 1000)
|
||||
return;
|
||||
|
||||
this._lastModified = performance.now();
|
||||
const lines: string[] = [];
|
||||
for (const action of actions) {
|
||||
const prefix = `${(++this._ordinal).toString().padStart(3, '0')}`;
|
||||
lines.push(
|
||||
`### User action: ${action.action.name}`,
|
||||
);
|
||||
if (action.code) {
|
||||
lines.push(
|
||||
`- Code`,
|
||||
'```js',
|
||||
action.code,
|
||||
'```');
|
||||
}
|
||||
if (action.action.ariaSnapshot) {
|
||||
const fileName = `${prefix}.snapshot.yml`;
|
||||
await fs.promises.writeFile(path.join(this._folder, fileName), action.action.ariaSnapshot);
|
||||
lines.push(`- Snapshot: ${fileName}`);
|
||||
}
|
||||
lines.push('', '', '');
|
||||
}
|
||||
|
||||
await this._appendLines(lines);
|
||||
}
|
||||
|
||||
private async _appendLines(lines: string[]) {
|
||||
await fs.promises.appendFile(this._file, lines.join('\n'));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +69,11 @@ export class Tab extends EventEmitter<TabEventsInterface> {
|
||||
});
|
||||
page.setDefaultNavigationTimeout(60000);
|
||||
page.setDefaultTimeout(5000);
|
||||
(page as any)[tabSymbol] = this;
|
||||
}
|
||||
|
||||
static forPage(page: playwright.Page): Tab | undefined {
|
||||
return (page as any)[tabSymbol];
|
||||
}
|
||||
|
||||
modalStates(): ModalState[] {
|
||||
@@ -308,3 +313,5 @@ function trim(text: string, maxLength: number) {
|
||||
return text;
|
||||
return text.slice(0, maxLength) + '...';
|
||||
}
|
||||
|
||||
const tabSymbol = Symbol('tabSymbol');
|
||||
|
||||
@@ -37,7 +37,6 @@ const uploadFile = defineTabTool({
|
||||
if (!modalState)
|
||||
throw new Error('No file chooser visible');
|
||||
|
||||
response.addCode(`// Select files for upload`);
|
||||
response.addCode(`await fileChooser.setFiles(${JSON.stringify(params.paths)})`);
|
||||
|
||||
tab.clearModalState(modalState);
|
||||
|
||||
@@ -60,9 +60,9 @@ export function defineTabTool<Input extends z.Schema>(tool: TabTool<Input>): Too
|
||||
const tab = context.currentTabOrDie();
|
||||
const modalStates = tab.modalStates().map(state => state.type);
|
||||
if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState))
|
||||
response.addError(`The tool "${tool.schema.name}" can only be used when there is related modal state present.\n` + tab.modalStatesMarkdown().join('\n'));
|
||||
response.addError(`Error: The tool "${tool.schema.name}" can only be used when there is related modal state present.\n` + tab.modalStatesMarkdown().join('\n'));
|
||||
else if (!tool.clearsModalState && modalStates.length)
|
||||
response.addError(`Tool "${tool.schema.name}" does not handle the modal state.\n` + tab.modalStatesMarkdown().join('\n'));
|
||||
response.addError(`Error: Tool "${tool.schema.name}" does not handle the modal state.\n` + tab.modalStatesMarkdown().join('\n'));
|
||||
else
|
||||
return tool.handle(tab, params, response);
|
||||
},
|
||||
|
||||
@@ -225,7 +225,7 @@ export function formatOutput(output: string): string[] {
|
||||
}
|
||||
|
||||
function parseResponse(response: any) {
|
||||
const text = (response as any).content[0].text;
|
||||
const text = response.content[0].text;
|
||||
const sections = parseSections(text);
|
||||
|
||||
const result = sections.get('Result');
|
||||
@@ -237,6 +237,7 @@ function parseResponse(response: any) {
|
||||
const downloads = sections.get('Downloads');
|
||||
const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, '');
|
||||
const isError = response.isError;
|
||||
const attachments = response.content.slice(1);
|
||||
|
||||
return {
|
||||
result,
|
||||
@@ -247,6 +248,7 @@ function parseResponse(response: any) {
|
||||
modalState,
|
||||
downloads,
|
||||
isError,
|
||||
attachments,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -31,18 +31,13 @@ test('browser_take_screenshot (viewport)', async ({ startClient, server }, testI
|
||||
|
||||
expect(await client.callTool({
|
||||
name: 'browser_take_screenshot',
|
||||
})).toEqual({
|
||||
content: [
|
||||
{
|
||||
text: expect.stringContaining(`Screenshot viewport and save it as`),
|
||||
type: 'text',
|
||||
},
|
||||
{
|
||||
data: expect.any(String),
|
||||
mimeType: 'image/jpeg',
|
||||
type: 'image',
|
||||
},
|
||||
],
|
||||
})).toHaveResponse({
|
||||
code: expect.stringContaining(`await page.screenshot`),
|
||||
attachments: [{
|
||||
data: expect.any(String),
|
||||
mimeType: 'image/jpeg',
|
||||
type: 'image',
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -207,7 +202,7 @@ test('browser_take_screenshot (imageResponses=omit)', async ({ startClient, serv
|
||||
})).toEqual({
|
||||
content: [
|
||||
{
|
||||
text: expect.stringContaining(`Screenshot viewport and save it as`),
|
||||
text: expect.stringContaining(`await page.screenshot`),
|
||||
type: 'text',
|
||||
},
|
||||
],
|
||||
@@ -231,7 +226,7 @@ test('browser_take_screenshot (fullPage: true)', async ({ startClient, server },
|
||||
})).toEqual({
|
||||
content: [
|
||||
{
|
||||
text: expect.stringContaining(`Screenshot full page and save it as`),
|
||||
text: expect.stringContaining('fullPage: true'),
|
||||
type: 'text',
|
||||
},
|
||||
{
|
||||
@@ -285,7 +280,7 @@ test('browser_take_screenshot (viewport without snapshot)', async ({ startClient
|
||||
})).toEqual({
|
||||
content: [
|
||||
{
|
||||
text: expect.stringContaining(`Screenshot viewport and save it as`),
|
||||
text: expect.stringContaining(`page.screenshot`),
|
||||
type: 'text',
|
||||
},
|
||||
{
|
||||
|
||||
153
tests/session-log.spec.ts
Normal file
153
tests/session-log.spec.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { test, expect } from './fixtures.js';
|
||||
|
||||
test('session log should record tool calls', async ({ startClient, server }, testInfo) => {
|
||||
const { client, stderr } = await startClient({
|
||||
args: [
|
||||
'--save-session',
|
||||
'--output-dir', testInfo.outputPath('output'),
|
||||
],
|
||||
});
|
||||
|
||||
server.setContent('/', `<title>Title</title><button>Submit</button>`, 'text/html');
|
||||
|
||||
await client.callTool({
|
||||
name: 'browser_navigate',
|
||||
arguments: { url: server.PREFIX },
|
||||
});
|
||||
|
||||
expect(await client.callTool({
|
||||
name: 'browser_click',
|
||||
arguments: {
|
||||
element: 'Submit button',
|
||||
ref: 'e2',
|
||||
},
|
||||
})).toHaveResponse({
|
||||
code: `await page.getByRole('button', { name: 'Submit' }).click();`,
|
||||
pageState: expect.stringContaining(`- button "Submit"`),
|
||||
});
|
||||
|
||||
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||
const sessionFolder = output.substring('Session: '.length);
|
||||
const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
|
||||
expect(sessionLog).toBe(`### Tool call: browser_navigate
|
||||
- Args
|
||||
\`\`\`json
|
||||
{
|
||||
"url": "http://localhost:${server.PORT}/"
|
||||
}
|
||||
\`\`\`
|
||||
- Code
|
||||
\`\`\`js
|
||||
await page.goto('http://localhost:${server.PORT}/');
|
||||
\`\`\`
|
||||
- Snapshot: 001.snapshot.yml
|
||||
|
||||
|
||||
### Tool call: browser_click
|
||||
- Args
|
||||
\`\`\`json
|
||||
{
|
||||
"element": "Submit button",
|
||||
"ref": "e2"
|
||||
}
|
||||
\`\`\`
|
||||
- Code
|
||||
\`\`\`js
|
||||
await page.getByRole('button', { name: 'Submit' }).click();
|
||||
\`\`\`
|
||||
- Snapshot: 002.snapshot.yml
|
||||
|
||||
|
||||
`);
|
||||
});
|
||||
|
||||
test('session log should record tool user actions', async ({ cdpServer, startClient }, testInfo) => {
|
||||
const browserContext = await cdpServer.start();
|
||||
const { client, stderr } = await startClient({
|
||||
args: [
|
||||
'--save-session',
|
||||
'--output-dir', testInfo.outputPath('output'),
|
||||
`--cdp-endpoint=${cdpServer.endpoint}`,
|
||||
],
|
||||
});
|
||||
|
||||
const [page] = browserContext.pages();
|
||||
await page.setContent(`
|
||||
<button>Button 1</button>
|
||||
<button>Button 2</button>
|
||||
`);
|
||||
|
||||
await client.callTool({
|
||||
name: 'browser_snapshot',
|
||||
});
|
||||
|
||||
// Manual action.
|
||||
await page.getByRole('button', { name: 'Button 1' }).click();
|
||||
|
||||
// This is to simulate a delay after the user action before the tool action.
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
// Tool action.
|
||||
await client.callTool({
|
||||
name: 'browser_click',
|
||||
arguments: {
|
||||
element: 'Button 2',
|
||||
ref: 'e3',
|
||||
},
|
||||
});
|
||||
|
||||
const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
|
||||
const sessionFolder = output.substring('Session: '.length);
|
||||
const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
|
||||
expect(sessionLog).toBe(`### Tool call: browser_snapshot
|
||||
- Args
|
||||
\`\`\`json
|
||||
{}
|
||||
\`\`\`
|
||||
- Snapshot: 001.snapshot.yml
|
||||
|
||||
|
||||
### User action: click
|
||||
- Code
|
||||
\`\`\`js
|
||||
await page.getByRole('button', { name: 'Button 1' }).click();
|
||||
\`\`\`
|
||||
- Snapshot: 002.snapshot.yml
|
||||
|
||||
|
||||
### Tool call: browser_click
|
||||
- Args
|
||||
\`\`\`json
|
||||
{
|
||||
"element": "Button 2",
|
||||
"ref": "e3"
|
||||
}
|
||||
\`\`\`
|
||||
- Code
|
||||
\`\`\`js
|
||||
await page.getByRole('button', { name: 'Button 2' }).click();
|
||||
\`\`\`
|
||||
- Snapshot: 003.snapshot.yml
|
||||
|
||||
|
||||
`);
|
||||
});
|
||||
Reference in New Issue
Block a user