feat(dialog): handle dialogs (#212)

src/tools/dialogs.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import type { ToolFactory } from './tool';
+
+const handleDialogSchema = z.object({
+  accept: z.boolean().describe('Whether to accept the dialog.'),
+  promptText: z.string().optional().describe('The text of the prompt in case of a prompt dialog.'),
+});
+
+const handleDialog: ToolFactory = captureSnapshot => ({
+  capability: 'core',
+
+  schema: {
+    name: 'browser_handle_dialog',
+    description: 'Handle a dialog',
+    inputSchema: zodToJsonSchema(handleDialogSchema),
+  },
+
+  handle: async (context, params) => {
+    const validatedParams = handleDialogSchema.parse(params);
+    const dialogState = context.modalStates().find(state => state.type === 'dialog');
+    if (!dialogState)
+      throw new Error('No dialog visible');
+
+    if (validatedParams.accept)
+      await dialogState.dialog.accept(validatedParams.promptText);
+    else
+      await dialogState.dialog.dismiss();
+
+    context.clearModalState(dialogState);
+
+    const code = [
+      `// <internal code to handle "${dialogState.dialog.type()}" dialog>`,
+    ];
+
+    return {
+      code,
+      captureSnapshot,
+      waitForNetwork: false,
+    };
+  },
+
+  clearsModalState: 'dialog',
+});
+
+export default (captureSnapshot: boolean) => [
+  handleDialog(captureSnapshot),
+];

src/tools/tool.ts

@@ -32,14 +32,22 @@ export type FileUploadModalState = {
   fileChooser: playwright.FileChooser;
 };
 
-export type ModalState = FileUploadModalState;
+export type DialogModalState = {
+  type: 'dialog';
+  description: string;
+  dialog: playwright.Dialog;
+};
+
+export type ModalState = FileUploadModalState | DialogModalState;
+
+export type ToolActionResult = { content?: (ImageContent | TextContent)[] } | undefined | void;
 
 export type ToolResult = {
   code: string[];
-  action?: () => Promise<{ content?: (ImageContent | TextContent)[] } | undefined | void>;
+  action?: () => Promise<ToolActionResult>;
   captureSnapshot: boolean;
   waitForNetwork: boolean;
-  resultOverride?: { content?: (ImageContent | TextContent)[] };
+  resultOverride?: ToolActionResult;
 };
 
 export type Tool = {

src/tools/utils.ts

@@ -15,8 +15,9 @@
  */
 
 import type * as playwright from 'playwright';
+import type { Context } from '../context';
 
-export async function waitForCompletion<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
+export async function waitForCompletion<R>(context: Context, page: playwright.Page, callback: () => Promise<R>): Promise<R> {
   const requests = new Set<playwright.Request>();
   let frameNavigated = false;
   let waitCallback: () => void = () => {};
@@ -62,7 +63,7 @@ export async function waitForCompletion<R>(page: playwright.Page, callback: () =
     if (!requests.size && !frameNavigated)
       waitCallback();
     await waitBarrier;
-    await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
+    await context.waitForTimeout(1000);
     return result;
   } finally {
     dispose();