Open Image in new tab in "debugMode"

.env.example

@@ -1 +1 @@
-VITE_API_SERVER_URL=http://localhost:8000
+VITE_DEBUG_MODE=true

src/common/App.tsx

@@ -1,11 +1,10 @@
-import { Box, ChakraProvider, Heading, HStack } from '@chakra-ui/react';
-import React from 'react';
-import { useAppState } from '../state/store';
-import ModelDropdown from './ModelDropdown';
-import SetAPIKey from './SetAPIKey';
-import TaskUI from './TaskUI';
-import OptionsDropdown from './OptionsDropdown';
-import logo from '../assets/img/icon-128.png';
+import { Box, ChakraProvider, Heading, HStack } from "@chakra-ui/react";
+import React from "react";
+import { useAppState } from "../state/store";
+import ModelDropdown from "./ModelDropdown";
+import SetAPIKey from "./SetAPIKey";
+import TaskUI from "./TaskUI";
+import OptionsDropdown from "./OptionsDropdown";
 
 const App = () => {
   const openAIKey = useAppState((state) => state.settings.openAIKey);

src/common/OptionsDropdown.tsx

@@ -1,14 +1,15 @@
-import { RepeatIcon, SettingsIcon, UnlockIcon } from '@chakra-ui/icons';
+import { RepeatIcon, SettingsIcon, UnlockIcon } from "@chakra-ui/icons";
 import {
   IconButton,
   Menu,
   MenuButton,
   MenuItem,
   MenuList,
-} from '@chakra-ui/react';
-import React from 'react';
-import { useAppState } from '../state/store';
-import { debugMode } from '../constants';
+} from "@chakra-ui/react";
+import React from "react";
+import { useAppState } from "../state/store";
+import { debugMode } from "../constants";
+import { findActiveTab } from "../helpers/browserUtils";
 
 const OptionsDropdown = () => {
   const { openAIKey, updateSettings } = useAppState((state) => ({
@@ -19,8 +20,15 @@ const OptionsDropdown = () => {
   if (!openAIKey) return null;
 
   const injectFunctions = async () => {
+    console.log("injecting functions");
+    const tab = await findActiveTab();
+    if (!tab) {
+      console.log("no active tab found");
+      return;
+    }
     chrome.runtime.sendMessage({
-      action: 'injectFunctions',
+      action: "injectFunctions",
+      tabId: tab.id,
     });
   };
 
@@ -36,7 +44,7 @@ const OptionsDropdown = () => {
         <MenuItem
           icon={<RepeatIcon />}
           onClick={() => {
-            updateSettings({ openAIKey: '' });
+            updateSettings({ openAIKey: "" });
           }}
         >
           Reset API Key

src/common/TaskUI.tsx

@@ -12,6 +12,7 @@ function ActionExecutor() {
     detachDegugger: state.currentTask.actions.detachDebugger,
     performActionString: state.currentTask.actions.performActionString,
     prepareLabels: state.currentTask.actions.prepareLabels,
+    showImagePrompt: state.currentTask.actions.showImagePrompt,
   }));
   const [action, setAction] = useState<string>(`{
   "thought": "try searching",
@@ -28,6 +29,7 @@ function ActionExecutor() {
       <HStack>
         <Button onClick={state.attachDebugger}>Attach</Button>
         <Button onClick={state.prepareLabels}>Prepare</Button>
+        <Button onClick={state.showImagePrompt}>Show Image</Button>
         <Button
           onClick={() => {
             state.performActionString(action);

src/constants.ts

@@ -1,7 +1,7 @@
-export const TAXY_ELEMENT_SELECTOR = 'data-taxy-node-id';
-export const VISIBLE_TEXT_ATTRIBUTE_NAME = 'data-web-wand-visible-text';
-export const ARIA_LABEL_ATTRIBUTE_NAME = 'data-web-wand-aria-label';
-export const WEB_WAND_LABEL_ATTRIBUTE_NAME = 'data-web-wand-label';
+export const TAXY_ELEMENT_SELECTOR = "data-taxy-node-id";
+export const VISIBLE_TEXT_ATTRIBUTE_NAME = "data-web-wand-visible-text";
+export const ARIA_LABEL_ATTRIBUTE_NAME = "data-web-wand-aria-label";
+export const WEB_WAND_LABEL_ATTRIBUTE_NAME = "data-web-wand-label";
 
 // read from env
-export const debugMode = process.env.DEBUG_MODE === 'true';
+export const debugMode = import.meta.env.VITE_DEBUG_MODE === "true";

src/helpers/browserUtils.ts

@@ -0,0 +1,15 @@
+export async function findActiveTab() {
+  const currentWindow = await chrome.windows.getCurrent();
+  if (!currentWindow || !currentWindow.id) {
+    throw new Error("Could not find window");
+  }
+  const tabs = await chrome.tabs.query({
+    active: true,
+    windowId: currentWindow.id,
+  });
+  const tab = tabs[0];
+  if (tab && tab.id != null) {
+    return tab;
+  }
+  return null;
+}

src/pages/background/index.ts

@@ -16,14 +16,17 @@ chrome.sidePanel
   .setPanelBehavior({ openPanelOnActionClick: true })
   .catch((error) => console.error(error));
 
-chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
-  if (message.action === "openNewTab") {
-    chrome.tabs.create({ url: message.url }, (tab) => {
-      // You can send back a response if needed
-      sendResponse({ status: "Tab opened", tabId: tab.id });
-    });
-
-    // Return true to indicate you wish to send a response asynchronously
+chrome.runtime.onMessage.addListener((message) => {
+  if (message.action === "injectFunctions") {
+    if (message.tabId == null) {
+      console.log("no active tab found");
+    } else {
+      chrome.scripting.executeScript({
+        target: { tabId: message.tabId },
+        files: ["assets/js/mainWorld.js"],
+        world: "MAIN",
+      });
+    }
     return true;
   }
 });

src/pages/content/domOperations.ts

@@ -34,6 +34,7 @@ export const initializeRPC = () => {
   chrome.runtime.onMessage.addListener(
     (message: RPCMessage, sender, sendResponse): true | undefined => {
       const { method, payload } = message;
+      console.log("RPC listener", method);
       if (method in rpcMethods) {
         // @ts-expect-error - we know this is valid (see pageRPC)
         const resp = rpcMethods[method as keyof RPCMethods](...payload);

src/pages/content/mainWorld/index.ts

@@ -0,0 +1,10 @@
+/**
+ * DO NOT USE import someModule from '...';
+ *
+ * @issue-url https://github.com/Jonghakseo/chrome-extension-boilerplate-react-vite/issues/160
+ *
+ * Chrome extensions don't support modules in content scripts.
+ * If you want to use other modules in content scripts, you need to import them via these files.
+ *
+ */
+import("@pages/content/mainWorld/mainWorld");

src/pages/content/mainWorld.ts → src/pages/content/mainWorld/mainWorld.ts

@@ -1,10 +1,10 @@
 // This file will be inject dynamically into the page as a content script running in the context of the page
 // see Background/index.ts for how this is done
 
-import { debugMode } from "../../constants";
-import { generateSimplifiedDom } from "../../helpers/simplifyDom";
-import getAnnotatedDOM from "./getAnnotatedDOM";
-import { rpcMethods } from "./domOperations";
+import { debugMode } from "@src/constants";
+import { generateSimplifiedDom } from "@src/helpers/simplifyDom";
+import getAnnotatedDOM from "../getAnnotatedDOM";
+import { rpcMethods } from "../domOperations";
 
 async function getSimplifiedDomFromPage() {
   const fullDom = getAnnotatedDOM();

src/state/currentTask.ts

@@ -18,28 +18,9 @@ import { callRPCWithTab } from "../helpers/rpc/pageRPC";
 import { getSimplifiedDom } from "../helpers/simplifyDom";
 import { sleep, truthyFilter } from "../helpers/utils";
 import performAction, { Action } from "../helpers/rpc/performAction";
+import { findActiveTab } from "../helpers/browserUtils";
 import { MyStateCreator, useAppState } from "./store";
 
-async function findActiveTab() {
-  const inspectedTabId = chrome?.devtools?.inspectedWindow?.tabId;
-  if (inspectedTabId) {
-    return await chrome.tabs.get(inspectedTabId);
-  }
-  const currentWindow = await chrome.windows.getCurrent();
-  if (!currentWindow || !currentWindow.id) {
-    throw new Error("Could not find window");
-  }
-  const tabs = await chrome.tabs.query({
-    active: true,
-    windowId: currentWindow.id,
-  });
-  const tab = tabs[0];
-  if (tab && tab.id != null) {
-    return tab;
-  }
-  return null;
-}
-
 export type TaskHistoryEntry = {
   prompt: string;
   response: string;
@@ -65,6 +46,7 @@ export type CurrentTaskSlice = {
     interrupt: () => void;
     attachDebugger: () => Promise<void>;
     detachDebugger: () => Promise<void>;
+    showImagePrompt: () => Promise<void>;
     prepareLabels: () => Promise<void>;
     performActionString: (actionString: string) => Promise<void>;
   };
@@ -243,6 +225,17 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
     detachDebugger: async () => {
       await detachDebugger(get().currentTask.tabId);
     },
+    showImagePrompt: async () => {
+      const tabId = get().currentTask.tabId;
+      await callRPCWithTab(tabId, "drawLabels", []);
+      await sleep(300);
+      const imgData = await chrome.tabs.captureVisibleTab({
+        format: "png",
+      });
+      openBase64InNewTab(imgData, "image/png");
+      await sleep(300);
+      await callRPCWithTab(tabId, "removeLabels", []);
+    },
     prepareLabels: async () => {
       const tabId = get().currentTask.tabId;
       await callRPCWithTab(tabId, "drawLabels", []);
@@ -272,3 +265,25 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
     },
   },
 });
+
+function openBase64InNewTab(base64Data: string, contentType: string) {
+  // Remove the prefix (e.g., "data:image/png;base64,") from the base64 data
+  const base64 = base64Data.split(";base64,").pop();
+  if (!base64) {
+    console.error("Invalid base64 data");
+    return;
+  }
+
+  // Convert base64 to a Blob
+  const byteCharacters = atob(base64);
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+  const byteArray = new Uint8Array(byteNumbers);
+  const blob = new Blob([byteArray], { type: contentType });
+
+  // Create a URL for the Blob and open it in a new tab
+  const blobUrl = URL.createObjectURL(blob);
+  window.open(blobUrl, "_blank");
+}

vite.config.ts

@@ -46,10 +46,13 @@ export default defineConfig({
       input: {
         devtools: resolve(pagesDir, "devtools", "index.html"),
         panel: resolve(pagesDir, "panel", "index.html"),
-        content: resolve(pagesDir, "content", "index.ts"),
         background: resolve(pagesDir, "background", "index.ts"),
-        contentStyle: resolve(pagesDir, "content", "style.scss"),
+        content: resolve(pagesDir, "content", "index.ts"),
         contentStyleGlobal: resolve(pagesDir, "content", "style.global.scss"),
+        contentStyle: resolve(pagesDir, "content", "style.scss"),
+        // TODO: current cannot support multiple content script entry files
+        // https://github.com/Jonghakseo/chrome-extension-boilerplate-react-vite/issues/306#issuecomment-1981885190
+        // mainWorld: resolve(pagesDir, "content/mainWorld", "index.ts"),
         popup: resolve(pagesDir, "popup", "index.html"),
         newtab: resolve(pagesDir, "newtab", "index.html"),
         options: resolve(pagesDir, "options", "index.html"),