diff --git a/config.d.ts b/config.d.ts index d63b061..af43236 100644 --- a/config.d.ts +++ b/config.d.ts @@ -16,7 +16,7 @@ import type * as playwright from 'playwright'; -export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf'; +export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf' | 'verify'; export type Config = { /** diff --git a/src/tools.ts b/src/tools.ts index 4f5d348..0eedf85 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -22,6 +22,7 @@ import files from './tools/files.js'; import form from './tools/form.js'; import install from './tools/install.js'; import keyboard from './tools/keyboard.js'; +import mouse from './tools/mouse.js'; import navigate from './tools/navigate.js'; import network from './tools/network.js'; import pdf from './tools/pdf.js'; @@ -29,7 +30,7 @@ import snapshot from './tools/snapshot.js'; import tabs from './tools/tabs.js'; import screenshot from './tools/screenshot.js'; import wait from './tools/wait.js'; -import mouse from './tools/mouse.js'; +import verify from './tools/verify.js'; import type { Tool } from './tools/tool.js'; import type { FullConfig } from './config.js'; @@ -51,6 +52,7 @@ export const allTools: Tool[] = [ ...snapshot, ...tabs, ...wait, + ...verify, ]; export function filteredTools(config: FullConfig) { diff --git a/src/tools/verify.ts b/src/tools/verify.ts new file mode 100644 index 0000000..881a1a4 --- /dev/null +++ b/src/tools/verify.ts @@ -0,0 +1,149 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'zod'; + +import { defineTabTool } from './tool.js'; +import * as javascript from '../utils/codegen.js'; +import { generateLocator } from './utils.js'; + +const verifyElement = defineTabTool({ + capability: 'verify', + schema: { + name: 'browser_verify_element_visible', + title: 'Verify element visible', + description: 'Verify element is visible on the page', + inputSchema: z.object({ + role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'), + accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'), + }), + type: 'readOnly', + }, + + handle: async (tab, params, response) => { + const locator = tab.page.getByRole(params.role as any, { name: params.accessibleName }); + if (await locator.count() === 0) { + response.addError(`Element with role "${params.role}" and accessible name "${params.accessibleName}" not found`); + return; + } + + response.addCode(`await expect(page.getByRole(${javascript.escapeWithQuotes(params.role)}, { name: ${javascript.escapeWithQuotes(params.accessibleName)} })).toBeVisible();`); + response.addResult('Done'); + }, +}); + +const verifyText = defineTabTool({ + capability: 'verify', + schema: { + name: 'browser_verify_text_visible', + title: 'Verify text visible', + description: `Verify text is visible on the page. Prefer ${verifyElement.schema.name} if possible.`, + inputSchema: z.object({ + text: z.string().describe('TEXT to verify. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'), + }), + type: 'readOnly', + }, + + handle: async (tab, params, response) => { + const locator = tab.page.getByText(params.text).filter({ visible: true }); + if (await locator.count() === 0) { + response.addError('Text not found'); + return; + } + + response.addCode(`await expect(page.getByText(${javascript.escapeWithQuotes(params.text)})).toBeVisible();`); + response.addResult('Done'); + }, +}); + +const verifyList = defineTabTool({ + capability: 'verify', + schema: { + name: 'browser_verify_list_visible', + title: 'Verify list visible', + description: 'Verify list is visible on the page', + inputSchema: z.object({ + element: z.string().describe('Human-readable list description'), + ref: z.string().describe('Exact target element reference that points to the list'), + items: z.array(z.string()).describe('Items to verify'), + }), + type: 'readOnly', + }, + + handle: async (tab, params, response) => { + const locator = await tab.refLocator({ ref: params.ref, element: params.element }); + const itemTexts: string[] = []; + for (const item of params.items) { + const itemLocator = locator.getByText(item); + if (await itemLocator.count() === 0) { + response.addError(`Item "${item}" not found`); + return; + } + itemTexts.push((await itemLocator.textContent())!); + } + const ariaSnapshot = `\` +- list: +${itemTexts.map(t => ` - text: ${javascript.escapeWithQuotes(t, '"')}`).join('\n')} +\``; + response.addCode(`await expect(page.locator('body')).toMatchAriaSnapshot(${ariaSnapshot});`); + response.addResult('Done'); + }, +}); + +const verifyValue = defineTabTool({ + capability: 'verify', + schema: { + name: 'browser_verify_value', + title: 'Verify value', + description: 'Verify element value', + inputSchema: z.object({ + type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'), + element: z.string().describe('Human-readable element description'), + ref: z.string().describe('Exact target element reference that points to the element'), + value: z.string().describe('Value to verify. For checkbox, use "true" or "false".'), + }), + type: 'readOnly', + }, + + handle: async (tab, params, response) => { + const locator = await tab.refLocator({ ref: params.ref, element: params.element }); + const locatorSource = `page.${await generateLocator(locator)}`; + if (params.type === 'textbox' || params.type === 'slider' || params.type === 'combobox') { + const value = await locator.inputValue(); + if (value !== params.value) { + response.addError(`Expected value "${params.value}", but got "${value}"`); + return; + } + response.addCode(`await expect(${locatorSource}).toHaveValue(${javascript.quote(params.value)});`); + } else if (params.type === 'checkbox' || params.type === 'radio') { + const value = await locator.isChecked(); + if (value !== (params.value === 'true')) { + response.addError(`Expected value "${params.value}", but got "${value}"`); + return; + } + const matcher = value ? 'toBeChecked' : 'not.toBeChecked'; + response.addCode(`await expect(${locatorSource}).${matcher}();`); + } + response.addResult('Done'); + }, +}); + +export default [ + verifyElement, + verifyText, + verifyList, + verifyValue, +]; diff --git a/tests/fixtures.ts b/tests/fixtures.ts index 966dc9d..d6368e8 100644 --- a/tests/fixtures.ts +++ b/tests/fixtures.ts @@ -31,6 +31,7 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'; import type { Stream } from 'stream'; export type TestOptions = { + mcpArgs: string[] | undefined; mcpBrowser: string | undefined; mcpMode: 'docker' | undefined; }; @@ -65,17 +66,19 @@ type WorkerFixtures = { export const test = baseTest.extend({ + mcpArgs: [undefined, { option: true }], + client: async ({ startClient }, use) => { const { client } = await startClient(); await use(client); }, - startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => { + startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => { const configDir = path.dirname(test.info().config.configFile!); const clients: Client[] = []; await use(async options => { - const args: string[] = []; + const args: string[] = mcpArgs ?? []; if (process.env.CI && process.platform === 'linux') args.push('--no-sandbox'); if (mcpHeadless) diff --git a/tests/verify.spec.ts b/tests/verify.spec.ts new file mode 100644 index 0000000..4831a92 --- /dev/null +++ b/tests/verify.spec.ts @@ -0,0 +1,522 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './fixtures.js'; + +test.use({ mcpArgs: ['--caps=verify'] }); + +test('browser_verify_element_visible', async ({ client, server }) => { + server.setContent('/', ` + Test Page + +

Welcome

+
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_element_visible', + arguments: { + role: 'button', + accessibleName: 'Submit', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByRole('button', { name: 'Submit' })).toBeVisible();`, + }); + + expect(await client.callTool({ + name: 'browser_verify_element_visible', + arguments: { + role: 'heading', + accessibleName: 'Welcome', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByRole('heading', { name: 'Welcome' })).toBeVisible();`, + }); + + expect(await client.callTool({ + name: 'browser_verify_element_visible', + arguments: { + role: 'alert', + accessibleName: 'Success message', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByRole('alert', { name: 'Success message' })).toBeVisible();`, + }); +}); + +test('browser_verify_element_visible (not found)', async ({ client, server }) => { + server.setContent('/', ` + Test Page + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_element_visible', + arguments: { + role: 'button', + accessibleName: 'Cancel', + }, + })).toHaveResponse({ + isError: true, + result: 'Element with role "button" and accessible name "Cancel" not found', + }); +}); + +test('browser_verify_text_visible', async ({ client, server }) => { + server.setContent('/', ` + Test Page +

Hello world

+
Welcome to our site
+ Status: Active + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: 'Hello world', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByText('Hello world')).toBeVisible();`, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: 'Welcome to our site', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByText('Welcome to our site')).toBeVisible();`, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: 'Status: Active', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByText('Status: Active')).toBeVisible();`, + }); +}); + +test('browser_verify_text_visible (not found)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +

Hello world

+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: 'Goodbye world', + }, + })).toHaveResponse({ + isError: true, + result: 'Text not found', + }); +}); + +test('browser_verify_text_visible (with quotes)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +

She said "Hello world"

+
It's a beautiful day
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: 'She said "Hello world"', + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByText('She said "Hello world"')).toBeVisible();`, + }); + + expect(await client.callTool({ + name: 'browser_verify_text_visible', + arguments: { + text: "It's a beautiful day", + }, + })).toHaveResponse({ + result: 'Done', + code: `await expect(page.getByText('It\\'s a beautiful day')).toBeVisible();`, + }); +}); + +test('browser_verify_list_visible', async ({ client, server }) => { + server.setContent('/', ` + Test Page + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_list_visible', + arguments: { + element: 'Fruit list', + ref: 'e2', + items: ['Apple', 'Banana', 'Cherry'], + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\` +- list: + - text: "Apple" + - text: "Banana" + - text: "Cherry" +\`);`), + }); +}); + +test('browser_verify_list_visible (partial items)', async ({ client, server }) => { + server.setContent('/', ` + Test Page + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_list_visible', + arguments: { + element: 'Fruit list', + ref: 'e2', + items: ['Apple', 'Cherry'], + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\` +- list: + - text: "Apple" + - text: "Cherry" +\`);`), + }); +}); + +test('browser_verify_list_visible (item not found)', async ({ client, server }) => { + server.setContent('/', ` + Test Page + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_list_visible', + arguments: { + element: 'Fruit list', + ref: 'e2', + items: ['Apple', 'Cherry'], + }, + })).toHaveResponse({ + isError: true, + result: 'Item "Cherry" not found', + }); +}); + +test('browser_verify_value (textbox)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'textbox', + element: 'Name textbox', + ref: 'e3', + value: 'John Doe', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Name' })).toHaveValue('John Doe');`), + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'textbox', + element: 'Email textbox', + ref: 'e4', + value: 'john@example.com', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Email' })).toHaveValue('john@example.com');`), + }); +}); + +test('browser_verify_value (textbox wrong value)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'textbox', + element: 'Name textbox', + ref: 'e3', + value: 'Jane Smith', + }, + })).toHaveResponse({ + isError: true, + result: 'Expected value "Jane Smith", but got "John Doe"', + }); +}); + +test('browser_verify_value (checkbox checked)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'checkbox', + element: 'Subscribe checkbox', + ref: 'e3', + value: 'true', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('checkbox')).toBeChecked();`), + }); +}); + +test('browser_verify_value (checkbox unchecked)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'checkbox', + element: 'Subscribe checkbox', + ref: 'e3', + value: 'false', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('checkbox')).not.toBeChecked();`), + }); +}); + +test('browser_verify_value (checkbox wrong value)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'checkbox', + element: 'Subscribe checkbox', + ref: 'e3', + value: 'false', + }, + })).toHaveResponse({ + isError: true, + result: 'Expected value "false", but got "true"', + }); +}); + +test('browser_verify_value (radio checked)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + + + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'radio', + element: 'Color radio', + ref: 'e4', + value: 'true', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('radio', { name: 'Red' })).toBeChecked();`), + }); +}); + +test('browser_verify_value (slider)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ + +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'slider', + element: 'Volume slider', + ref: 'e3', + value: '75', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('slider')).toHaveValue('75');`), + }); +}); + +test('browser_verify_value (combobox)', async ({ client, server }) => { + server.setContent('/', ` + Test Page +
+ +
+ `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_verify_value', + arguments: { + type: 'combobox', + element: 'Country select', + ref: 'e3', + value: 'United States', + }, + })).toHaveResponse({ + result: 'Done', + code: expect.stringContaining(`await expect(page.getByRole('combobox')).toHaveValue('United States');`), + }); +});