From 408d8e6e66a680dc13cb46b75cf23dd79621a2da Mon Sep 17 00:00:00 2001 From: mondaychen Date: Mon, 18 Mar 2024 11:54:36 -0400 Subject: [PATCH 1/3] fix: better coverage for airbnb & healthgrade --- src/helpers/knowledge/db.json | 2 +- src/pages/content/drawLabels.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/knowledge/db.json b/src/helpers/knowledge/db.json index dddd446..f8b9604 100644 --- a/src/helpers/knowledge/db.json +++ b/src/helpers/knowledge/db.json @@ -51,7 +51,7 @@ "airbnb.com": { "rules": [ { - "regexes": ["^/?$"], + "regexes": [".*"], "knowledge": { "annotationRules": [ { diff --git a/src/pages/content/drawLabels.ts b/src/pages/content/drawLabels.ts index 9bff267..fb394e1 100644 --- a/src/pages/content/drawLabels.ts +++ b/src/pages/content/drawLabels.ts @@ -10,7 +10,7 @@ styleElem.setAttribute("type", "text/css"); styleElem.innerHTML = ` ._label_overlay_wrapper { position: absolute; - z-index: 9999; + z-index: 99999999; top: 0; left: 0; } From b156f452daf4372e7b1c539f61131c856341f751 Mon Sep 17 00:00:00 2001 From: mondaychen Date: Mon, 18 Mar 2024 16:14:18 -0400 Subject: [PATCH 2/3] fix: adjust between-page animation order --- manifest.js | 1 + src/helpers/rpc/domActions.ts | 3 ++- src/pages/content/ripple.ts | 9 ++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/manifest.js b/manifest.js index f26d017..8fd5384 100755 --- a/manifest.js +++ b/manifest.js @@ -42,6 +42,7 @@ const manifest = { matches: ["http://*/*", "https://*/*", ""], js: ["src/pages/content/index.js"], css: ["assets/css/contentStyleGlobal.css"], + run_at: "document_start", // load the js as soon as possible since it does not rely on the DOM }, ], // devtools_page: "src/pages/devtools/index.html", diff --git a/src/helpers/rpc/domActions.ts b/src/helpers/rpc/domActions.ts index b166c55..1daa048 100644 --- a/src/helpers/rpc/domActions.ts +++ b/src/helpers/rpc/domActions.ts @@ -78,7 +78,7 @@ export class DomActions { y: number, clickCount = 1, ): Promise { - callRPCWithTab(this.tabId, "ripple", [x, y]); + await callRPCWithTab(this.tabId, "ripple", [x, y]); await this.sendCommand("Input.dispatchMouseEvent", { type: "mousePressed", x, @@ -86,6 +86,7 @@ export class DomActions { button: "left", clickCount, }); + await sleep(20); await this.sendCommand("Input.dispatchMouseEvent", { type: "mouseReleased", x, diff --git a/src/pages/content/ripple.ts b/src/pages/content/ripple.ts index d88b97e..04d746f 100644 --- a/src/pages/content/ripple.ts +++ b/src/pages/content/ripple.ts @@ -1,6 +1,6 @@ import { sleep } from "../../helpers/utils"; -export default async function ripple(x: number, y: number) { +export default function ripple(x: number, y: number) { const rippleRadius = 30; const ripple = document.createElement("div"); ripple.classList.add("web-agent-ripple"); @@ -11,6 +11,9 @@ export default async function ripple(x: number, y: number) { document.body.appendChild(ripple); - await sleep(1000); - ripple.remove(); + // remove after the animation to finish + // but we don't need to `await` it + sleep(800).then(() => { + ripple.remove(); + }); } From 85895f4507829542f6858e24e7ab1bfea69b1a67 Mon Sep 17 00:00:00 2001 From: mondaychen Date: Mon, 18 Mar 2024 16:15:01 -0400 Subject: [PATCH 3/3] feat: improve correctness on Amazon --- src/helpers/knowledge/db.json | 22 ++++++++++++++++++++++ src/helpers/knowledge/redirects.json | 3 ++- src/pages/content/drawLabels.ts | 4 ++-- src/pages/content/style.global.scss | 2 +- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/helpers/knowledge/db.json b/src/helpers/knowledge/db.json index f8b9604..f53ecd1 100644 --- a/src/helpers/knowledge/db.json +++ b/src/helpers/knowledge/db.json @@ -64,5 +64,27 @@ } } ] + }, + "amazon.com": { + "rules": [ + { + "regexes": [".*"], + "knowledge": { + "notes": [ + "Be careful not to confuse the \"Add to Cart\" with the \"Buy Now\". The \"Add to Cart\" button adds the item to the cart, while the \"Buy Now\" button takes you to the checkout page.", + "Do not confuse \"Buy Now\" and \"Buy New\" on product page.", + "You should always verify if the product is in stock before buying or adding to the cart. If the product is not in stock, you should notify the user about it." + ] + } + }, + { + "regexes": ["^/s$"], + "knowledge": { + "notes": [ + "There is no \"Add to Cart\" button on the search results page. You need to click on the product name to go to the product details page first." + ] + } + } + ] } } diff --git a/src/helpers/knowledge/redirects.json b/src/helpers/knowledge/redirects.json index 1bb5f2b..266022a 100644 --- a/src/helpers/knowledge/redirects.json +++ b/src/helpers/knowledge/redirects.json @@ -2,5 +2,6 @@ "twitter.com": "x.com", "www.twitter.com": "x.com", "www.x.com": "x.com", - "www.airbnb.com": "airbnb.com" + "www.airbnb.com": "airbnb.com", + "www.amazon.com": "amazon.com" } diff --git a/src/pages/content/drawLabels.ts b/src/pages/content/drawLabels.ts index fb394e1..d3a0699 100644 --- a/src/pages/content/drawLabels.ts +++ b/src/pages/content/drawLabels.ts @@ -68,8 +68,8 @@ function isVisible( const { display, visibility, opacity } = style; if (display === "none") return false; if (visibility !== "visible") return false; - if (opacity != null && parseFloat(opacity) < 0.1) return false; // considering something very close to 0 as invisible - // don't respect 'aria-hidden' attribute since they are most likely visible to the user + if (opacity != null && parseFloat(opacity) <= 0) return false; + // don't respect 'aria-hidden' attribute since they are most likely visible in the screenshot // if (element.getAttribute('aria-hidden') === 'true') return false; const rect = element.getBoundingClientRect(); diff --git a/src/pages/content/style.global.scss b/src/pages/content/style.global.scss index 6de434e..aae2cf5 100644 --- a/src/pages/content/style.global.scss +++ b/src/pages/content/style.global.scss @@ -8,7 +8,7 @@ animation: web-agent-ripple 0.5s ease-out; background-color: rgba(0, 0, 255, 0.961); pointer-events: none; - z-index: 9999; + z-index: 999999; } @keyframes web-agent-ripple {