diff --git a/src/helpers/rpc/performAction.ts b/src/helpers/rpc/performAction.ts index 5b0403d..d0caccb 100644 --- a/src/helpers/rpc/performAction.ts +++ b/src/helpers/rpc/performAction.ts @@ -179,6 +179,20 @@ function createOperateTool( } break; } + case "setValueAndSubmit": { + const success = await setValue( + domActions, + action.args.label, + (action.args.value || "") + "\n", + ); + if (!success) { + console.error( + "Unable to find element with label: ", + action.args.label, + ); + } + break; + } default: console.error("Invalid action name", action); } diff --git a/src/helpers/vision-agent/tools.ts b/src/helpers/vision-agent/tools.ts index 5a1f113..d272137 100644 --- a/src/helpers/vision-agent/tools.ts +++ b/src/helpers/vision-agent/tools.ts @@ -22,6 +22,18 @@ export const setValueSchema = z.object({ value: z.string(), }), }); +export const setValueAndSubmitSchema = z.object({ + name: z.literal("setValueAndSubmit"), + description: z + .literal( + 'Like "setValue", except then it presses ENTER to submit the form.', + ) + .optional(), + args: z.object({ + label: z.string(), + value: z.string(), + }), +}); export const navigateSchema = z.object({ name: z.literal("navigate"), @@ -74,6 +86,7 @@ export const failSchema = z.object({ export const toolSchemaUnion = z.discriminatedUnion("name", [ clickSchema, setValueSchema, + setValueAndSubmitSchema, navigateSchema, scrollSchema, waitSchema,