chore: initial code commit

2025-03-21 10:58:58 -07:00
parent b1d5410a1b
commit 852709c026
23 changed files with 6307 additions and 204 deletions
--- a/src/tools/common.ts
+++ b/src/tools/common.ts
@@ -0,0 +1,165 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import path from 'path';
+
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import { captureAriaSnapshot, runAndWait } from './utils';
+
+import type { ToolFactory, Tool } from './tool';
+
+const navigateSchema = z.object({
+  url: z.string().describe('The URL to navigate to'),
+});
+
+export const navigate: ToolFactory = snapshot => ({
+  schema: {
+    name: 'browser_navigate',
+    description: 'Navigate to a URL',
+    inputSchema: zodToJsonSchema(navigateSchema),
+  },
+  handle: async (context, params) => {
+    const validatedParams = navigateSchema.parse(params);
+    const page = await context.ensurePage();
+    await page.goto(validatedParams.url, { waitUntil: 'domcontentloaded' });
+    // Cap load event to 5 seconds, the page is operational at this point.
+    await page.waitForLoadState('load', { timeout: 5000 }).catch(() => {});
+    if (snapshot)
+      return captureAriaSnapshot(page);
+    return {
+      content: [{
+        type: 'text',
+        text: `Navigated to ${validatedParams.url}`,
+      }],
+    };
+  },
+});
+
+const goBackSchema = z.object({});
+
+export const goBack: ToolFactory = snapshot => ({
+  schema: {
+    name: 'browser_go_back',
+    description: 'Go back to the previous page',
+    inputSchema: zodToJsonSchema(goBackSchema),
+  },
+  handle: async context => {
+    return await runAndWait(context, 'Navigated back', async () => {
+      const page = await context.ensurePage();
+      await page.goBack();
+    }, snapshot);
+  },
+});
+
+const goForwardSchema = z.object({});
+
+export const goForward: ToolFactory = snapshot => ({
+  schema: {
+    name: 'browser_go_forward',
+    description: 'Go forward to the next page',
+    inputSchema: zodToJsonSchema(goForwardSchema),
+  },
+  handle: async context => {
+    return await runAndWait(context, 'Navigated forward', async () => {
+      const page = await context.ensurePage();
+      await page.goForward();
+    }, snapshot);
+  },
+});
+
+const waitSchema = z.object({
+  time: z.number().describe('The time to wait in seconds'),
+});
+
+export const wait: Tool = {
+  schema: {
+    name: 'browser_wait',
+    description: 'Wait for a specified time in seconds',
+    inputSchema: zodToJsonSchema(waitSchema),
+  },
+  handle: async (context, params) => {
+    const validatedParams = waitSchema.parse(params);
+    const page = await context.ensurePage();
+    await page.waitForTimeout(Math.min(10000, validatedParams.time * 1000));
+    return {
+      content: [{
+        type: 'text',
+        text: `Waited for ${validatedParams.time} seconds`,
+      }],
+    };
+  },
+};
+
+const pressKeySchema = z.object({
+  key: z.string().describe('Name of the key to press or a character to generate, such as `ArrowLeft` or `a`'),
+});
+
+export const pressKey: Tool = {
+  schema: {
+    name: 'browser_press_key',
+    description: 'Press a key on the keyboard',
+    inputSchema: zodToJsonSchema(pressKeySchema),
+  },
+  handle: async (context, params) => {
+    const validatedParams = pressKeySchema.parse(params);
+    return await runAndWait(context, `Pressed key ${validatedParams.key}`, async page => {
+      await page.keyboard.press(validatedParams.key);
+    });
+  },
+};
+
+const pdfSchema = z.object({});
+
+export const pdf: Tool = {
+  schema: {
+    name: 'browser_save_as_pdf',
+    description: 'Save page as PDF',
+    inputSchema: zodToJsonSchema(pdfSchema),
+  },
+  handle: async context => {
+    const page = await context.ensurePage();
+    const fileName = path.join(os.tmpdir(), `/page-${new Date().toISOString()}.pdf`);
+    await page.pdf({ path: fileName });
+    return {
+      content: [{
+        type: 'text',
+        text: `Saved as ${fileName}`,
+      }],
+    };
+  },
+};
+
+const closeSchema = z.object({});
+
+export const close: Tool = {
+  schema: {
+    name: 'browser_close',
+    description: 'Close the page',
+    inputSchema: zodToJsonSchema(closeSchema),
+  },
+  handle: async context => {
+    await context.close();
+    return {
+      content: [{
+        type: 'text',
+        text: `Page closed`,
+      }],
+    };
+  },
+};
--- a/src/tools/screenshot.ts
+++ b/src/tools/screenshot.ts
@@ -0,0 +1,133 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import { runAndWait } from './utils';
+
+import type { Tool } from './tool';
+
+export const screenshot: Tool = {
+  schema: {
+    name: 'browser_screenshot',
+    description: 'Take a screenshot of the current page',
+    inputSchema: zodToJsonSchema(z.object({})),
+  },
+
+  handle: async context => {
+    const page = await context.ensurePage();
+    const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });
+    return {
+      content: [{ type: 'image', data: screenshot.toString('base64'), mimeType: 'image/jpeg' }],
+    };
+  },
+};
+
+const elementSchema = z.object({
+  element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
+});
+
+const moveMouseSchema = elementSchema.extend({
+  x: z.number().describe('X coordinate'),
+  y: z.number().describe('Y coordinate'),
+});
+
+export const moveMouse: Tool = {
+  schema: {
+    name: 'browser_move_mouse',
+    description: 'Move mouse to a given position',
+    inputSchema: zodToJsonSchema(moveMouseSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = moveMouseSchema.parse(params);
+    const page = await context.ensurePage();
+    await page.mouse.move(validatedParams.x, validatedParams.y);
+    return {
+      content: [{ type: 'text', text: `Moved mouse to (${validatedParams.x}, ${validatedParams.y})` }],
+    };
+  },
+};
+
+const clickSchema = elementSchema.extend({
+  x: z.number().describe('X coordinate'),
+  y: z.number().describe('Y coordinate'),
+});
+
+export const click: Tool = {
+  schema: {
+    name: 'browser_click',
+    description: 'Click left mouse button',
+    inputSchema: zodToJsonSchema(clickSchema),
+  },
+
+  handle: async (context, params) => {
+    return await runAndWait(context, 'Clicked mouse', async page => {
+      const validatedParams = clickSchema.parse(params);
+      await page.mouse.move(validatedParams.x, validatedParams.y);
+      await page.mouse.down();
+      await page.mouse.up();
+    });
+  },
+};
+
+const dragSchema = elementSchema.extend({
+  startX: z.number().describe('Start X coordinate'),
+  startY: z.number().describe('Start Y coordinate'),
+  endX: z.number().describe('End X coordinate'),
+  endY: z.number().describe('End Y coordinate'),
+});
+
+export const drag: Tool = {
+  schema: {
+    name: 'browser_drag',
+    description: 'Drag left mouse button',
+    inputSchema: zodToJsonSchema(dragSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = dragSchema.parse(params);
+    return await runAndWait(context, `Dragged mouse from (${validatedParams.startX}, ${validatedParams.startY}) to (${validatedParams.endX}, ${validatedParams.endY})`, async page => {
+      await page.mouse.move(validatedParams.startX, validatedParams.startY);
+      await page.mouse.down();
+      await page.mouse.move(validatedParams.endX, validatedParams.endY);
+      await page.mouse.up();
+    });
+  },
+};
+
+const typeSchema = z.object({
+  text: z.string().describe('Text to type into the element'),
+  submit: z.boolean().describe('Whether to submit entered text (press Enter after)'),
+});
+
+export const type: Tool = {
+  schema: {
+    name: 'browser_type',
+    description: 'Type text',
+    inputSchema: zodToJsonSchema(typeSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = typeSchema.parse(params);
+    return await runAndWait(context, `Typed text "${validatedParams.text}"`, async page => {
+      await page.keyboard.type(validatedParams.text);
+      if (validatedParams.submit)
+        await page.keyboard.press('Enter');
+    });
+  },
+};
--- a/src/tools/snapshot.ts
+++ b/src/tools/snapshot.ts
@@ -0,0 +1,117 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+import zodToJsonSchema from 'zod-to-json-schema';
+
+import { captureAriaSnapshot, runAndWait } from './utils';
+
+import type * as playwright from 'playwright';
+import type { Tool } from './tool';
+
+export const snapshot: Tool = {
+  schema: {
+    name: 'browser_snapshot',
+    description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
+    inputSchema: zodToJsonSchema(z.object({})),
+  },
+
+  handle: async context => {
+    return await captureAriaSnapshot(await context.ensurePage());
+  },
+};
+
+const elementSchema = z.object({
+  element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
+  ref: z.string().describe('Exact target element reference from the page snapshot'),
+});
+
+export const click: Tool = {
+  schema: {
+    name: 'browser_click',
+    description: 'Perform click on a web page',
+    inputSchema: zodToJsonSchema(elementSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = elementSchema.parse(params);
+    return runAndWait(context, `"${validatedParams.element}" clicked`, page => refLocator(page, validatedParams.ref).click(), true);
+  },
+};
+
+const dragSchema = z.object({
+  startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
+  startRef: z.string().describe('Exact source element reference from the page snapshot'),
+  endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
+  endRef: z.string().describe('Exact target element reference from the page snapshot'),
+});
+
+export const drag: Tool = {
+  schema: {
+    name: 'browser_drag',
+    description: 'Perform drag and drop between two elements',
+    inputSchema: zodToJsonSchema(dragSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = dragSchema.parse(params);
+    return runAndWait(context, `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`, async page => {
+      const startLocator = refLocator(page, validatedParams.startRef);
+      const endLocator = refLocator(page, validatedParams.endRef);
+      await startLocator.dragTo(endLocator);
+    }, true);
+  },
+};
+
+export const hover: Tool = {
+  schema: {
+    name: 'browser_hover',
+    description: 'Hover over element on page',
+    inputSchema: zodToJsonSchema(elementSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = elementSchema.parse(params);
+    return runAndWait(context, `Hovered over "${validatedParams.element}"`, page => refLocator(page, validatedParams.ref).hover(), true);
+  },
+};
+
+const typeSchema = elementSchema.extend({
+  text: z.string().describe('Text to type into the element'),
+  submit: z.boolean().describe('Whether to submit entered text (press Enter after)'),
+});
+
+export const type: Tool = {
+  schema: {
+    name: 'browser_type',
+    description: 'Type text into editable element',
+    inputSchema: zodToJsonSchema(typeSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = typeSchema.parse(params);
+    return await runAndWait(context, `Typed "${validatedParams.text}" into "${validatedParams.element}"`, async page => {
+      const locator = refLocator(page, validatedParams.ref);
+      await locator.fill(validatedParams.text);
+      if (validatedParams.submit)
+        await locator.press('Enter');
+    }, true);
+  },
+};
+
+function refLocator(page: playwright.Page, ref: string): playwright.Locator {
+  return page.locator(`aria-ref=${ref}`);
+}
--- a/src/tools/tool.ts
+++ b/src/tools/tool.ts
@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types';
+import type { JsonSchema7Type } from 'zod-to-json-schema';
+import type { Context } from '../context';
+
+export type ToolSchema = {
+  name: string;
+  description: string;
+  inputSchema: JsonSchema7Type;
+};
+
+export type ToolResult = {
+  content: (ImageContent | TextContent)[];
+  isError?: boolean;
+};
+
+export type Tool = {
+  schema: ToolSchema;
+  handle: (context: Context, params?: Record<string, any>) => Promise<ToolResult>;
+};
+
+export type ToolFactory = (snapshot: boolean) => Tool;
--- a/src/tools/utils.ts
+++ b/src/tools/utils.ts
@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as playwright from 'playwright';
+import type { ToolResult } from './tool';
+import type { Context } from '../context';
+
+async function waitForCompletion<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
+  const requests = new Set<playwright.Request>();
+  let frameNavigated = false;
+  let waitCallback: () => void = () => {};
+  const waitBarrier = new Promise<void>(f => { waitCallback = f; });
+
+  const requestListener = (request: playwright.Request) => requests.add(request);
+  const requestFinishedListener = (request: playwright.Request) => {
+    requests.delete(request);
+    if (!requests.size)
+      waitCallback();
+  };
+
+  const frameNavigateListener = (frame: playwright.Frame) => {
+    if (frame.parentFrame())
+      return;
+    frameNavigated = true;
+    dispose();
+    clearTimeout(timeout);
+    void frame.waitForLoadState('load').then(() => {
+      waitCallback();
+    });
+  };
+
+  const onTimeout = () => {
+    dispose();
+    waitCallback();
+  };
+
+  page.on('request', requestListener);
+  page.on('requestfinished', requestFinishedListener);
+  page.on('framenavigated', frameNavigateListener);
+  const timeout = setTimeout(onTimeout, 10000);
+
+  const dispose = () => {
+    page.off('request', requestListener);
+    page.off('requestfinished', requestFinishedListener);
+    page.off('framenavigated', frameNavigateListener);
+    clearTimeout(timeout);
+  };
+
+  try {
+    const result = await callback();
+    if (!requests.size && !frameNavigated)
+      waitCallback();
+    await waitBarrier;
+    await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
+    return result;
+  } finally {
+    dispose();
+  }
+}
+
+export async function runAndWait(context: Context, status: string, callback: (page: playwright.Page) => Promise<any>, snapshot: boolean = false): Promise<ToolResult> {
+  const page = await context.ensurePage();
+  await waitForCompletion(page, () => callback(page));
+  return snapshot ? captureAriaSnapshot(page, status) : {
+    content: [{ type: 'text', text: status }],
+  };
+}
+
+export async function captureAriaSnapshot(page: playwright.Page, status: string = ''): Promise<ToolResult> {
+  const snapshot = await page.locator('html').ariaSnapshot({ ref: true });
+  return {
+    content: [{ type: 'text', text: `${status ? `${status}\n` : ''}
+- Page URL: ${page.url()}
+- Page Title: ${await page.title()}
+- Page Snapshot
+\`\`\`yaml
+${snapshot}
+\`\`\`
+`
+    }],
+  };
+}