chore: record user actions in the session log (#798)

tests/fixtures.ts

@@ -225,7 +225,7 @@ export function formatOutput(output: string): string[] {
 }
 
 function parseResponse(response: any) {
-  const text = (response as any).content[0].text;
+  const text = response.content[0].text;
   const sections = parseSections(text);
 
   const result = sections.get('Result');
@@ -237,6 +237,7 @@ function parseResponse(response: any) {
   const downloads = sections.get('Downloads');
   const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, '');
   const isError = response.isError;
+  const attachments = response.content.slice(1);
 
   return {
     result,
@@ -247,6 +248,7 @@ function parseResponse(response: any) {
     modalState,
     downloads,
     isError,
+    attachments,
   };
 }
 

tests/screenshot.spec.ts

@@ -31,18 +31,13 @@ test('browser_take_screenshot (viewport)', async ({ startClient, server }, testI
 
   expect(await client.callTool({
     name: 'browser_take_screenshot',
-  })).toEqual({
-    content: [
-      {
-        text: expect.stringContaining(`Screenshot viewport and save it as`),
-        type: 'text',
-      },
-      {
-        data: expect.any(String),
-        mimeType: 'image/jpeg',
-        type: 'image',
-      },
-    ],
+  })).toHaveResponse({
+    code: expect.stringContaining(`await page.screenshot`),
+    attachments: [{
+      data: expect.any(String),
+      mimeType: 'image/jpeg',
+      type: 'image',
+    }],
   });
 });
 
@@ -207,7 +202,7 @@ test('browser_take_screenshot (imageResponses=omit)', async ({ startClient, serv
   })).toEqual({
     content: [
       {
-        text: expect.stringContaining(`Screenshot viewport and save it as`),
+        text: expect.stringContaining(`await page.screenshot`),
         type: 'text',
       },
     ],
@@ -231,7 +226,7 @@ test('browser_take_screenshot (fullPage: true)', async ({ startClient, server },
   })).toEqual({
     content: [
       {
-        text: expect.stringContaining(`Screenshot full page and save it as`),
+        text: expect.stringContaining('fullPage: true'),
         type: 'text',
       },
       {
@@ -285,7 +280,7 @@ test('browser_take_screenshot (viewport without snapshot)', async ({ startClient
   })).toEqual({
     content: [
       {
-        text: expect.stringContaining(`Screenshot viewport and save it as`),
+        text: expect.stringContaining(`page.screenshot`),
         type: 'text',
       },
       {

tests/session-log.spec.ts

@@ -0,0 +1,153 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import { test, expect } from './fixtures.js';
+
+test('session log should record tool calls', async ({ startClient, server }, testInfo) => {
+  const { client, stderr } = await startClient({
+    args: [
+      '--save-session',
+      '--output-dir', testInfo.outputPath('output'),
+    ],
+  });
+
+  server.setContent('/', `<title>Title</title><button>Submit</button>`, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_click',
+    arguments: {
+      element: 'Submit button',
+      ref: 'e2',
+    },
+  })).toHaveResponse({
+    code: `await page.getByRole('button', { name: 'Submit' }).click();`,
+    pageState: expect.stringContaining(`- button "Submit"`),
+  });
+
+  const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
+  const sessionFolder = output.substring('Session: '.length);
+  const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
+  expect(sessionLog).toBe(`### Tool call: browser_navigate
+- Args
+\`\`\`json
+{
+  "url": "http://localhost:${server.PORT}/"
+}
+\`\`\`
+- Code
+\`\`\`js
+await page.goto('http://localhost:${server.PORT}/');
+\`\`\`
+- Snapshot: 001.snapshot.yml
+
+
+### Tool call: browser_click
+- Args
+\`\`\`json
+{
+  "element": "Submit button",
+  "ref": "e2"
+}
+\`\`\`
+- Code
+\`\`\`js
+await page.getByRole('button', { name: 'Submit' }).click();
+\`\`\`
+- Snapshot: 002.snapshot.yml
+
+
+`);
+});
+
+test('session log should record tool user actions', async ({ cdpServer, startClient }, testInfo) => {
+  const browserContext = await cdpServer.start();
+  const { client, stderr } = await startClient({
+    args: [
+      '--save-session',
+      '--output-dir', testInfo.outputPath('output'),
+      `--cdp-endpoint=${cdpServer.endpoint}`,
+    ],
+  });
+
+  const [page] = browserContext.pages();
+  await page.setContent(`
+    <button>Button 1</button>
+    <button>Button 2</button>
+  `);
+
+  await client.callTool({
+    name: 'browser_snapshot',
+  });
+
+  // Manual action.
+  await page.getByRole('button', { name: 'Button 1' }).click();
+
+  // This is to simulate a delay after the user action before the tool action.
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  // Tool action.
+  await client.callTool({
+    name: 'browser_click',
+    arguments: {
+      element: 'Button 2',
+      ref: 'e3',
+    },
+  });
+
+  const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0];
+  const sessionFolder = output.substring('Session: '.length);
+  const sessionLog = await fs.promises.readFile(path.join(sessionFolder, 'session.md'), 'utf8');
+  expect(sessionLog).toBe(`### Tool call: browser_snapshot
+- Args
+\`\`\`json
+{}
+\`\`\`
+- Snapshot: 001.snapshot.yml
+
+
+### User action: click
+- Code
+\`\`\`js
+await page.getByRole('button', { name: 'Button 1' }).click();
+\`\`\`
+- Snapshot: 002.snapshot.yml
+
+
+### Tool call: browser_click
+- Args
+\`\`\`json
+{
+  "element": "Button 2",
+  "ref": "e3"
+}
+\`\`\`
+- Code
+\`\`\`js
+await page.getByRole('button', { name: 'Button 2' }).click();
+\`\`\`
+- Snapshot: 003.snapshot.yml
+
+
+`);
+});