chore: introduce verification tools (#951)

config.d.ts

@@ -16,7 +16,7 @@
 
 import type * as playwright from 'playwright';
 
-export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf';
+export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf' | 'verify';
 
 export type Config = {
   /**

src/tools.ts

@@ -22,6 +22,7 @@ import files from './tools/files.js';
 import form from './tools/form.js';
 import install from './tools/install.js';
 import keyboard from './tools/keyboard.js';
+import mouse from './tools/mouse.js';
 import navigate from './tools/navigate.js';
 import network from './tools/network.js';
 import pdf from './tools/pdf.js';
@@ -29,7 +30,7 @@ import snapshot from './tools/snapshot.js';
 import tabs from './tools/tabs.js';
 import screenshot from './tools/screenshot.js';
 import wait from './tools/wait.js';
-import mouse from './tools/mouse.js';
+import verify from './tools/verify.js';
 
 import type { Tool } from './tools/tool.js';
 import type { FullConfig } from './config.js';
@@ -51,6 +52,7 @@ export const allTools: Tool<any>[] = [
   ...snapshot,
   ...tabs,
   ...wait,
+  ...verify,
 ];
 
 export function filteredTools(config: FullConfig) {

src/tools/verify.ts

@@ -0,0 +1,149 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTabTool } from './tool.js';
+import * as javascript from '../utils/codegen.js';
+import { generateLocator } from './utils.js';
+
+const verifyElement = defineTabTool({
+  capability: 'verify',
+  schema: {
+    name: 'browser_verify_element_visible',
+    title: 'Verify element visible',
+    description: 'Verify element is visible on the page',
+    inputSchema: z.object({
+      role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'),
+      accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const locator = tab.page.getByRole(params.role as any, { name: params.accessibleName });
+    if (await locator.count() === 0) {
+      response.addError(`Element with role "${params.role}" and accessible name "${params.accessibleName}" not found`);
+      return;
+    }
+
+    response.addCode(`await expect(page.getByRole(${javascript.escapeWithQuotes(params.role)}, { name: ${javascript.escapeWithQuotes(params.accessibleName)} })).toBeVisible();`);
+    response.addResult('Done');
+  },
+});
+
+const verifyText = defineTabTool({
+  capability: 'verify',
+  schema: {
+    name: 'browser_verify_text_visible',
+    title: 'Verify text visible',
+    description: `Verify text is visible on the page. Prefer ${verifyElement.schema.name} if possible.`,
+    inputSchema: z.object({
+      text: z.string().describe('TEXT to verify. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const locator = tab.page.getByText(params.text).filter({ visible: true });
+    if (await locator.count() === 0) {
+      response.addError('Text not found');
+      return;
+    }
+
+    response.addCode(`await expect(page.getByText(${javascript.escapeWithQuotes(params.text)})).toBeVisible();`);
+    response.addResult('Done');
+  },
+});
+
+const verifyList = defineTabTool({
+  capability: 'verify',
+  schema: {
+    name: 'browser_verify_list_visible',
+    title: 'Verify list visible',
+    description: 'Verify list is visible on the page',
+    inputSchema: z.object({
+      element: z.string().describe('Human-readable list description'),
+      ref: z.string().describe('Exact target element reference that points to the list'),
+      items: z.array(z.string()).describe('Items to verify'),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const locator = await tab.refLocator({ ref: params.ref, element: params.element });
+    const itemTexts: string[] = [];
+    for (const item of params.items) {
+      const itemLocator = locator.getByText(item);
+      if (await itemLocator.count() === 0) {
+        response.addError(`Item "${item}" not found`);
+        return;
+      }
+      itemTexts.push((await itemLocator.textContent())!);
+    }
+    const ariaSnapshot = `\`
+- list:
+${itemTexts.map(t => `  - text: ${javascript.escapeWithQuotes(t, '"')}`).join('\n')}
+\``;
+    response.addCode(`await expect(page.locator('body')).toMatchAriaSnapshot(${ariaSnapshot});`);
+    response.addResult('Done');
+  },
+});
+
+const verifyValue = defineTabTool({
+  capability: 'verify',
+  schema: {
+    name: 'browser_verify_value',
+    title: 'Verify value',
+    description: 'Verify element value',
+    inputSchema: z.object({
+      type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'),
+      element: z.string().describe('Human-readable element description'),
+      ref: z.string().describe('Exact target element reference that points to the element'),
+      value: z.string().describe('Value to verify. For checkbox, use "true" or "false".'),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const locator = await tab.refLocator({ ref: params.ref, element: params.element });
+    const locatorSource = `page.${await generateLocator(locator)}`;
+    if (params.type === 'textbox' || params.type === 'slider' || params.type === 'combobox') {
+      const value = await locator.inputValue();
+      if (value !== params.value) {
+        response.addError(`Expected value "${params.value}", but got "${value}"`);
+        return;
+      }
+      response.addCode(`await expect(${locatorSource}).toHaveValue(${javascript.quote(params.value)});`);
+    } else if (params.type === 'checkbox' || params.type === 'radio') {
+      const value = await locator.isChecked();
+      if (value !== (params.value === 'true')) {
+        response.addError(`Expected value "${params.value}", but got "${value}"`);
+        return;
+      }
+      const matcher = value ? 'toBeChecked' : 'not.toBeChecked';
+      response.addCode(`await expect(${locatorSource}).${matcher}();`);
+    }
+    response.addResult('Done');
+  },
+});
+
+export default [
+  verifyElement,
+  verifyText,
+  verifyList,
+  verifyValue,
+];

tests/fixtures.ts

@@ -31,6 +31,7 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
 import type { Stream } from 'stream';
 
 export type TestOptions = {
+  mcpArgs: string[] | undefined;
   mcpBrowser: string | undefined;
   mcpMode: 'docker' | undefined;
 };
@@ -65,17 +66,19 @@ type WorkerFixtures = {
 
 export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>({
 
+  mcpArgs: [undefined, { option: true }],
+
   client: async ({ startClient }, use) => {
     const { client } = await startClient();
     await use(client);
   },
 
-  startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => {
+  startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => {
     const configDir = path.dirname(test.info().config.configFile!);
     const clients: Client[] = [];
 
     await use(async options => {
-      const args: string[] = [];
+      const args: string[] = mcpArgs ?? [];
       if (process.env.CI && process.platform === 'linux')
         args.push('--no-sandbox');
       if (mcpHeadless)

tests/verify.spec.ts

@@ -0,0 +1,522 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './fixtures.js';
+
+test.use({ mcpArgs: ['--caps=verify'] });
+
+test('browser_verify_element_visible', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <button>Submit</button>
+    <h1>Welcome</h1>
+    <div role="alert" aria-label="Success message"></div>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_element_visible',
+    arguments: {
+      role: 'button',
+      accessibleName: 'Submit',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByRole('button', { name: 'Submit' })).toBeVisible();`,
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_element_visible',
+    arguments: {
+      role: 'heading',
+      accessibleName: 'Welcome',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByRole('heading', { name: 'Welcome' })).toBeVisible();`,
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_element_visible',
+    arguments: {
+      role: 'alert',
+      accessibleName: 'Success message',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByRole('alert', { name: 'Success message' })).toBeVisible();`,
+  });
+});
+
+test('browser_verify_element_visible (not found)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <button>Submit</button>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_element_visible',
+    arguments: {
+      role: 'button',
+      accessibleName: 'Cancel',
+    },
+  })).toHaveResponse({
+    isError: true,
+    result: 'Element with role "button" and accessible name "Cancel" not found',
+  });
+});
+
+test('browser_verify_text_visible', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <p>Hello world</p>
+    <div>Welcome to our site</div>
+    <span>Status: Active</span>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: 'Hello world',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByText('Hello world')).toBeVisible();`,
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: 'Welcome to our site',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByText('Welcome to our site')).toBeVisible();`,
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: 'Status: Active',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByText('Status: Active')).toBeVisible();`,
+  });
+});
+
+test('browser_verify_text_visible (not found)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <p>Hello world</p>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: 'Goodbye world',
+    },
+  })).toHaveResponse({
+    isError: true,
+    result: 'Text not found',
+  });
+});
+
+test('browser_verify_text_visible (with quotes)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <p>She said "Hello world"</p>
+    <div>It's a beautiful day</div>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: 'She said "Hello world"',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByText('She said "Hello world"')).toBeVisible();`,
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_text_visible',
+    arguments: {
+      text: "It's a beautiful day",
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: `await expect(page.getByText('It\\'s a beautiful day')).toBeVisible();`,
+  });
+});
+
+test('browser_verify_list_visible', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <ul>
+      <li>Apple</li>
+      <li>Banana</li>
+      <li>Cherry</li>
+    </ul>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_list_visible',
+    arguments: {
+      element: 'Fruit list',
+      ref: 'e2',
+      items: ['Apple', 'Banana', 'Cherry'],
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\`
+- list:
+  - text: "Apple"
+  - text: "Banana"
+  - text: "Cherry"
+\`);`),
+  });
+});
+
+test('browser_verify_list_visible (partial items)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <ul>
+      <li>Apple</li>
+      <li>Banana</li>
+      <li>Cherry</li>
+      <li>Date</li>
+    </ul>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_list_visible',
+    arguments: {
+      element: 'Fruit list',
+      ref: 'e2',
+      items: ['Apple', 'Cherry'],
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\`
+- list:
+  - text: "Apple"
+  - text: "Cherry"
+\`);`),
+  });
+});
+
+test('browser_verify_list_visible (item not found)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <ul>
+      <li>Apple</li>
+      <li>Banana</li>
+    </ul>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_list_visible',
+    arguments: {
+      element: 'Fruit list',
+      ref: 'e2',
+      items: ['Apple', 'Cherry'],
+    },
+  })).toHaveResponse({
+    isError: true,
+    result: 'Item "Cherry" not found',
+  });
+});
+
+test('browser_verify_value (textbox)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="text" aria-label="Name" value="John Doe" />
+      <input type="email" aria-label="Email" value="john@example.com" />
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'textbox',
+      element: 'Name textbox',
+      ref: 'e3',
+      value: 'John Doe',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Name' })).toHaveValue('John Doe');`),
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'textbox',
+      element: 'Email textbox',
+      ref: 'e4',
+      value: 'john@example.com',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Email' })).toHaveValue('john@example.com');`),
+  });
+});
+
+test('browser_verify_value (textbox wrong value)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="text" name="name" value="John Doe" />
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'textbox',
+      element: 'Name textbox',
+      ref: 'e3',
+      value: 'Jane Smith',
+    },
+  })).toHaveResponse({
+    isError: true,
+    result: 'Expected value "Jane Smith", but got "John Doe"',
+  });
+});
+
+test('browser_verify_value (checkbox checked)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="checkbox" name="subscribe" checked />
+      <label for="subscribe">Subscribe to newsletter</label>
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'checkbox',
+      element: 'Subscribe checkbox',
+      ref: 'e3',
+      value: 'true',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('checkbox')).toBeChecked();`),
+  });
+});
+
+test('browser_verify_value (checkbox unchecked)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="checkbox" name="subscribe" />
+      <label for="subscribe">Subscribe to newsletter</label>
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'checkbox',
+      element: 'Subscribe checkbox',
+      ref: 'e3',
+      value: 'false',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('checkbox')).not.toBeChecked();`),
+  });
+});
+
+test('browser_verify_value (checkbox wrong value)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="checkbox" name="subscribe" checked />
+      <label for="subscribe">Subscribe to newsletter</label>
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'checkbox',
+      element: 'Subscribe checkbox',
+      ref: 'e3',
+      value: 'false',
+    },
+  })).toHaveResponse({
+    isError: true,
+    result: 'Expected value "false", but got "true"',
+  });
+});
+
+test('browser_verify_value (radio checked)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <label for="red">Red</label>
+      <input id="red" type="radio" name="color" value="red" checked />
+      <label for="blue">Blue</label>
+      <input id="blue" type="radio" name="color" value="blue" />
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'radio',
+      element: 'Color radio',
+      ref: 'e4',
+      value: 'true',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('radio', { name: 'Red' })).toBeChecked();`),
+  });
+});
+
+test('browser_verify_value (slider)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <input type="range" name="volume" min="0" max="100" value="75" />
+      <label>Volume</label>
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'slider',
+      element: 'Volume slider',
+      ref: 'e3',
+      value: '75',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('slider')).toHaveValue('75');`),
+  });
+});
+
+test('browser_verify_value (combobox)', async ({ client, server }) => {
+  server.setContent('/', `
+    <title>Test Page</title>
+    <form>
+      <select name="country">
+        <option>Choose a country</option>
+        <option selected>United States</option>
+        <option>United Kingdom</option>
+      </select>
+    </form>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_verify_value',
+    arguments: {
+      type: 'combobox',
+      element: 'Country select',
+      ref: 'e3',
+      value: 'United States',
+    },
+  })).toHaveResponse({
+    result: 'Done',
+    code: expect.stringContaining(`await expect(page.getByRole('combobox')).toHaveValue('United States');`),
+  });
+});