Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support custom selector #35

Merged
merged 3 commits into from
Mar 7, 2024
Merged

feat: support custom selector #35

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f06cde1
feat: support custom selector
lynchee-owo Mar 6, 2024
de6157b
fix: updated lib version
lynchee-owo Mar 7, 2024
f8b2cfb
fix: improve code quality
lynchee-owo Mar 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.lib.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "web-wand-lib",
"version": "2.0.0-beta-1",
"version": "2.0.1",
"description": "Helper library for Web Wand",
"repository": {
"type": "git",
Expand Down
36 changes: 25 additions & 11 deletions src/helpers/parseResponse.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { ActionPayload, availableActions } from './availableActions';
import { useAppState } from "../state/store";
import {
ActionPayload,
availableActions,
availableActionsVision,
} from "./availableActions";

export type ParsedResponseSuccess = {
thought: string;
Expand All @@ -21,9 +26,9 @@ export function extractJsonFromMarkdown(input: string): string[] {
let match;
while ((match = regex.exec(input)) !== null) {
// If 'json' is specified, add the content to the results array
if (match[1] === 'json') {
if (match[1] === "json") {
results.push(match[2]);
} else if (match[2].startsWith('{')) {
} else if (match[2].startsWith("{")) {
results.push(match[2]);
}
}
Expand All @@ -36,8 +41,8 @@ function parseFunctionCall(callString: string) {
const matches = callString.match(functionPattern);

if (!matches) {
console.error('Input does not match a function call pattern.', callString);
throw new Error('Input does not match a function call pattern.');
console.error("Input does not match a function call pattern.", callString);
throw new Error("Input does not match a function call pattern.");
}

const [, name, argsPart] = matches;
Expand Down Expand Up @@ -70,26 +75,28 @@ function parseFunctionCall(callString: string) {
}

export function parseResponse(text: string): ParsedResponse {
const isVisionModel =
useAppState.getState().settings.selectedModel === "gpt-4-vision-preview";
let action;
lynchee-owo marked this conversation as resolved.
Show resolved Hide resolved
try {
action = JSON.parse(text);
} catch (_e) {
try {
action = JSON.parse(extractJsonFromMarkdown(text)[0]);
} catch (_e) {
throw new Error('Response does not contain valid JSON.');
throw new Error("Response does not contain valid JSON.");
}
}

if (!action.thought) {
return {
error: 'Invalid response: Thought not found in the model response.',
error: "Invalid response: Thought not found in the model response.",
};
}

if (!action.action) {
return {
error: 'Invalid response: Action not found in the model response.',
error: "Invalid response: Action not found in the model response.",
};
}

Expand All @@ -99,9 +106,16 @@ export function parseResponse(text: string): ParsedResponse {
const { name: actionName, args: argsArray } = parseFunctionCall(actionString);
console.log(actionName, argsArray);

const availableAction = availableActions.find(
(action) => action.name === actionName
);
let availableAction = null;
if (isVisionModel) {
availableAction = availableActionsVision.find(
(action) => action.name === actionName,
);
} else {
availableAction = availableActions.find(
(action) => action.name === actionName,
);
}

if (!availableAction) {
return {
Expand Down
192 changes: 136 additions & 56 deletions src/helpers/rpc/performAction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,52 @@ import {
VISIBLE_TEXT_ATTRIBUTE_NAME,
} from "../../constants";

function getSelector(selectorName: string): string {
function getLabel(selectorName: string): string {
return `[${WEB_WAND_LABEL_ATTRIBUTE_NAME}="${selectorName}"]`;
}

function getFallbackSelector(selectorName: string): string {
return `[${VISIBLE_TEXT_ATTRIBUTE_NAME}="${selectorName}"]`;
}

export type Action = {
name: "click" | "setValue" | "scroll" | "finish";
export type ActionName = "click" | "setValue" | "scroll" | "finish";

export type ActionWithLabel = {
name: ActionName;
args: {
text?: string;
label?: string;
label: string;
value?: string;
elementId?: string;
};
};

export type ActionWithElementId = {
name: ActionName;
args: {
elementId: string;
value?: string;
};
};

export type ActionWithSelector = {
name: ActionName;
args: {
selector: string;
value?: string;
};
};

export type Action = ActionWithLabel | ActionWithElementId | ActionWithSelector;

async function clickWithSelector(
domActions: DomActions,
selector: string,
): Promise<boolean> {
console.log("clickWithSelector", selector);
return await domActions.clickWithSelector({
selector,
});
}

async function clickWithElementId(
domActions: DomActions,
selectorName: string,
Expand All @@ -32,11 +60,11 @@ async function clickWithElementId(
});
}

async function clickWithSelector(
async function clickWithLabel(
domActions: DomActions,
selectorName: string,
): Promise<boolean> {
console.log("clickWithSelector", selectorName);
console.log("clickWithLabel", selectorName);
let success = false;
try {
success = await domActions.clickWithSelector({
Expand All @@ -47,14 +75,26 @@ async function clickWithSelector(
}
if (success) return true;
success = await domActions.clickWithSelector({
selector: getSelector(selectorName),
selector: getLabel(selectorName),
lynchee-owo marked this conversation as resolved.
Show resolved Hide resolved
});
if (success) return true;
return await domActions.clickWithSelector({
selector: getFallbackSelector(selectorName),
});
}

async function setValueWithSelector(
domActions: DomActions,
selector: string,
value: string,
): Promise<boolean> {
console.log("setValueWithSelector", selector);
return await domActions.setValueWithSelector({
selector,
value,
});
}

async function setValueWithElementId(
domActions: DomActions,
selectorName: string,
Expand All @@ -67,12 +107,12 @@ async function setValueWithElementId(
});
}

async function setValueWithSelector(
async function setValueWithLabel(
domActions: DomActions,
selectorName: string,
value: string,
): Promise<boolean> {
console.log("setValueWithSelector", selectorName);
console.log("setValueWithLabel", selectorName);
let success = false;
try {
success = await domActions.setValueWithSelector({
Expand All @@ -84,7 +124,7 @@ async function setValueWithSelector(
}
if (success) return true;
success = await domActions.setValueWithSelector({
selector: getSelector(selectorName),
selector: getLabel(selectorName),
value,
});
if (success) return true;
Expand All @@ -94,62 +134,102 @@ async function setValueWithSelector(
});
}

export default async function performAction(tabId: number, action: Action) {
console.log("performAction", tabId, action);
const domActions = new DomActions(tabId);
async function scroll(domActions: DomActions, action: Action) {
if (action.args.value === "up") {
await domActions.scrollUp();
} else {
await domActions.scrollDown();
}
}

export async function performActionWithSelector(
domActions: DomActions,
action: ActionWithSelector,
) {
const selectorName = action.args.selector;
if (action.name === "click") {
let selectorName = "";
let success = false;
if (action.args.elementId) {
selectorName = action.args.elementId;
success = await clickWithElementId(domActions, selectorName);
}
if (!success && action.args.label) {
selectorName = action.args.label;
success = await clickWithSelector(domActions, selectorName);
const success = await clickWithSelector(domActions, selectorName);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
if (!success && action.args.text) {
selectorName = action.args.text;
success = await clickWithSelector(domActions, selectorName);
} else if (action.name === "setValue") {
const success = await setValueWithSelector(
domActions,
selectorName,
action.args.value || "",
);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
} else if (action.name === "scroll") {
scroll(domActions, action);
} else {
console.log("other actions");
}
}

export async function performActionWithElementId(
domActions: DomActions,
action: ActionWithElementId,
) {
const selectorName = action.args.elementId;
if (action.name === "click") {
const success = await clickWithElementId(domActions, selectorName);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
} else if (action.name === "setValue") {
let selectorName = "";
let success = false;
if (action.args.elementId) {
selectorName = action.args.elementId;
success = await setValueWithElementId(
domActions,
selectorName,
action.args.value || "",
);
}
if (!success && action.args.label) {
selectorName = action.args.label;
success = await setValueWithSelector(
domActions,
selectorName,
action.args.value || "",
);
const success = await setValueWithElementId(
domActions,
selectorName,
action.args.value || "",
);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
if (!success && action.args.text) {
selectorName = action.args.text;
success = await setValueWithSelector(
domActions,
selectorName,
action.args.value || "",
);
} else if (action.name === "scroll") {
scroll(domActions, action);
} else {
console.log("other actions");
}
}

export async function performActionWithLabel(
domActions: DomActions,
action: ActionWithLabel,
) {
const selectorName = action.args.label;
if (action.name === "click") {
const success = await clickWithLabel(domActions, selectorName);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
} else if (action.name === "setValue") {
const success = await setValueWithLabel(
domActions,
selectorName,
action.args.value || "",
);
if (!success) {
console.error("Unable to find element with selector: ", selectorName);
}
} else if (action.name === "scroll") {
if (action.args.value === "up") {
await domActions.scrollUp();
} else {
await domActions.scrollDown();
}
scroll(domActions, action);
} else {
console.log("other actions");
}
}

export default async function performAction(tabId: number, action: Action) {
console.log("performAction", tabId, action);
const domActions = new DomActions(tabId);
if ("selector" in action.args) {
lynchee-owo marked this conversation as resolved.
Show resolved Hide resolved
await performActionWithSelector(domActions, action as ActionWithSelector);
} else if ("elementId" in action.args) {
await performActionWithElementId(domActions, action as ActionWithElementId);
} else if ("label" in action.args) {
await performActionWithLabel(domActions, action as ActionWithLabel);
} else {
console.error("Invalid action arguments", action);
}
}
Loading
Loading