feat: press enter to submit after typing

normal-computing · Mar 22, 2024 · acf4847 · acf4847
1 parent 425a973
commit acf4847
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 0 deletions.
diff --git a/src/helpers/rpc/performAction.ts b/src/helpers/rpc/performAction.ts
@@ -179,6 +179,20 @@ function createOperateTool(
         }
         break;
       }
+      case "setValueAndSubmit": {
+        const success = await setValue(
+          domActions,
+          action.args.label,
+          (action.args.value || "") + "\n",
+        );
+        if (!success) {
+          console.error(
+            "Unable to find element with label: ",
+            action.args.label,
+          );
+        }
+        break;
+      }
       default:
         console.error("Invalid action name", action);
     }

diff --git a/src/helpers/vision-agent/tools.ts b/src/helpers/vision-agent/tools.ts
@@ -22,6 +22,18 @@ export const setValueSchema = z.object({
     value: z.string(),
   }),
 });
+export const setValueAndSubmitSchema = z.object({
+  name: z.literal("setValueAndSubmit"),
+  description: z
+    .literal(
+      'Like "setValue", except then it presses ENTER to submit the form.',
+    )
+    .optional(),
+  args: z.object({
+    label: z.string(),
+    value: z.string(),
+  }),
+});
 
 export const navigateSchema = z.object({
   name: z.literal("navigate"),
@@ -74,6 +86,7 @@ export const failSchema = z.object({
 export const toolSchemaUnion = z.discriminatedUnion("name", [
   clickSchema,
   setValueSchema,
+  setValueAndSubmitSchema,
   navigateSchema,
   scrollSchema,
   waitSchema,