From 7903a73c757b9a9161af685e0737f6cb5976b648 Mon Sep 17 00:00:00 2001
From: Mengdi Chen <monday.chen@gmail.com>
Date: Tue, 23 Apr 2024 10:38:11 -0400
Subject: [PATCH] Simplify Instructions & show in main UI (#130)

* refactor: remove unused state

* refactor: rename notes to instructions & hide json mode

* feat: show matched notes in main UI

* fix: skip knowledge display if no notes

* fix: tsc issue
---
 src/common/App.tsx                            |  2 +-
 .../CustomKnowledgeBase/DefaultKnowledge.tsx  |  4 +-
 .../CustomKnowledgeBase/HostKnowledge.tsx     | 93 ++++++++++++-------
 .../CustomKnowledgeBase/MatchedNotes.tsx      | 55 +++++++++++
 .../CustomKnowledgeBase/NewKnowledgeForm.tsx  | 22 ++---
 src/common/CustomKnowledgeBase/Notes.tsx      | 14 +++
 src/common/CustomKnowledgeBase/index.tsx      | 22 ++---
 src/common/Settings.tsx                       |  4 +-
 src/common/TaskUI.tsx                         |  2 +
 src/state/currentTask.ts                      | 11 ++-
 10 files changed, 163 insertions(+), 66 deletions(-)
 create mode 100644 src/common/CustomKnowledgeBase/MatchedNotes.tsx
 create mode 100644 src/common/CustomKnowledgeBase/Notes.tsx

diff --git a/src/common/App.tsx b/src/common/App.tsx
index 954d201..857dda0 100644
--- a/src/common/App.tsx
+++ b/src/common/App.tsx
@@ -23,7 +23,7 @@ const App = () => {
 
   return (
     <ChakraProvider>
-      <Box p="8" pb="16" fontSize="lg" w="full">
+      <Box p="8" pb="24" fontSize="lg" w="full">
         <HStack mb={4} alignItems="center">
           <Heading as="h1" size="lg" flex={1}>
             WebWand 🪄
diff --git a/src/common/CustomKnowledgeBase/DefaultKnowledge.tsx b/src/common/CustomKnowledgeBase/DefaultKnowledge.tsx
index 861d47b..9d3f980 100644
--- a/src/common/CustomKnowledgeBase/DefaultKnowledge.tsx
+++ b/src/common/CustomKnowledgeBase/DefaultKnowledge.tsx
@@ -20,7 +20,7 @@ const DefaultKnowledge = () => {
   return (
     <>
       <Button size="sm" variant="link" colorScheme="blue" onClick={onOpen}>
-        View Default Knowledge
+        View Built-in Instructions
       </Button>
       <Modal
         isOpen={isOpen}
@@ -30,7 +30,7 @@ const DefaultKnowledge = () => {
       >
         <ModalOverlay />
         <ModalContent>
-          <ModalHeader>Default Knowledge Base</ModalHeader>
+          <ModalHeader>Default Instructions</ModalHeader>
           <ModalCloseButton />
           <ModalBody>
             {Object.keys(defaultKnowledgeBase).map((host) => (
diff --git a/src/common/CustomKnowledgeBase/HostKnowledge.tsx b/src/common/CustomKnowledgeBase/HostKnowledge.tsx
index 52dd6c2..862bef4 100644
--- a/src/common/CustomKnowledgeBase/HostKnowledge.tsx
+++ b/src/common/CustomKnowledgeBase/HostKnowledge.tsx
@@ -1,4 +1,4 @@
-import { DeleteIcon, CopyIcon, EditIcon } from "@chakra-ui/icons";
+import { DeleteIcon, EditIcon } from "@chakra-ui/icons";
 import {
   Heading,
   Accordion,
@@ -9,11 +9,11 @@ import {
   IconButton,
   Tooltip,
   Flex,
-  useToast,
   Box,
 } from "@chakra-ui/react";
 import { fetchAllDefaultKnowledge } from "../../helpers/knowledge";
 import { useAppState } from "@root/src/state/store";
+import Notes from "./Notes";
 
 type HostKnowledgeProps = {
   host: string;
@@ -26,7 +26,6 @@ const HostKnowledge = ({
   isDefaultKnowledge,
   onEdit,
 }: HostKnowledgeProps) => {
-  const toast = useToast();
   const updateSettings = useAppState((state) => state.settings.actions.update);
   const customKnowledgeBase = useAppState(
     (state) => state.settings.customKnowledgeBase,
@@ -35,9 +34,20 @@ const HostKnowledge = ({
     ? fetchAllDefaultKnowledge()
     : customKnowledgeBase;
 
-  const getJsonString = (): string => {
-    return JSON.stringify(knowledgeBase[host], null, 2);
-  };
+  if (knowledgeBase[host] === undefined) {
+    return null;
+  }
+  const rules = knowledgeBase[host].rules;
+  if (rules === undefined) {
+    return null;
+  }
+  const hasNotes = rules.some(
+    (rule) => (rule.knowledge?.notes?.length ?? 0) > 0,
+  );
+  // skip if no notes
+  if (!hasNotes) {
+    return null;
+  }
 
   const handleRemove = () => {
     const newKnowledge = { ...knowledgeBase };
@@ -45,6 +55,12 @@ const HostKnowledge = ({
     updateSettings({ customKnowledgeBase: newKnowledge });
   };
 
+  // temporarily disable copy feature
+  /*
+  const getJsonString = (): string => {
+    return JSON.stringify(knowledgeBase[host], null, 2);
+  };
+
   const handleCopy = async () => {
     try {
       await navigator.clipboard.writeText(getJsonString());
@@ -65,13 +81,23 @@ const HostKnowledge = ({
       });
     }
   };
+  */
 
   return (
     <>
-      <Flex alignItems="flex-start">
-        <Heading as="h5" size="sm" flex="1" overflowWrap="anywhere">
+      <Flex alignItems="flex-start" mb="2">
+        <Heading
+          as="h5"
+          size="sm"
+          flex="1"
+          overflowWrap="anywhere"
+          lineHeight="1.5rem"
+        >
           {!isDefaultKnowledge && (
-            <Box position="relative" style={{ float: "right" }}>
+            <Box
+              position="relative"
+              style={{ float: "right", marginTop: "-4px" }}
+            >
               <Tooltip label="Edit knowledge">
                 <IconButton
                   aria-label="Edit knowledge"
@@ -83,15 +109,6 @@ const HostKnowledge = ({
                   }}
                 />
               </Tooltip>
-              <Tooltip label="Copy knowledge">
-                <IconButton
-                  aria-label="Copy knowledge"
-                  icon={<CopyIcon />}
-                  size="sm"
-                  variant="ghost"
-                  onClick={handleCopy}
-                />
-              </Tooltip>
               <Tooltip label="Remove knowledge">
                 <IconButton
                   aria-label="Remove knowledge"
@@ -107,21 +124,31 @@ const HostKnowledge = ({
         </Heading>
       </Flex>
       <Accordion allowToggle>
-        {knowledgeBase[host].rules?.map((rule, ruleIndex) => (
-          <AccordionItem key={ruleIndex} backgroundColor="white">
-            <h2>
-              <AccordionButton>
-                <Box flex="1" textAlign="left">
-                  Rule {ruleIndex + 1}
-                </Box>
-                <AccordionIcon />
-              </AccordionButton>
-            </h2>
-            <AccordionPanel pb={4}>
-              <pre style={{ overflowX: "auto" }}>{getJsonString()}</pre>
-            </AccordionPanel>
-          </AccordionItem>
-        ))}
+        {rules.map((rule, ruleIndex) => {
+          // Skip rules without notes
+          if (
+            rule.knowledge === undefined ||
+            rule.knowledge.notes === undefined ||
+            rule.knowledge.notes.length === 0
+          ) {
+            return null;
+          }
+          return (
+            <AccordionItem key={ruleIndex} backgroundColor="white">
+              <h2>
+                <AccordionButton>
+                  <Box flex="1" textAlign="left">
+                    Instructions Set {ruleIndex + 1}
+                  </Box>
+                  <AccordionIcon />
+                </AccordionButton>
+              </h2>
+              <AccordionPanel pb={4}>
+                <Notes notes={rule.knowledge.notes} />
+              </AccordionPanel>
+            </AccordionItem>
+          );
+        })}
       </Accordion>
     </>
   );
diff --git a/src/common/CustomKnowledgeBase/MatchedNotes.tsx b/src/common/CustomKnowledgeBase/MatchedNotes.tsx
new file mode 100644
index 0000000..06e410e
--- /dev/null
+++ b/src/common/CustomKnowledgeBase/MatchedNotes.tsx
@@ -0,0 +1,55 @@
+import { useAppState } from "../../state/store";
+import {
+  Alert,
+  AlertIcon,
+  AlertDescription,
+  Button,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalHeader,
+  ModalOverlay,
+  useDisclosure,
+} from "@chakra-ui/react";
+import Notes from "./Notes";
+
+export default function MatchedNotes() {
+  const { isOpen, onOpen, onClose } = useDisclosure();
+  const knowledge = useAppState((state) => state.currentTask.knowledgeInUse);
+  const notes = knowledge?.notes;
+  if (!notes || notes.length === 0) {
+    return null;
+  }
+
+  return (
+    <>
+      <Button variant="link" onClick={onOpen}>
+        Found {notes.length} instructions.
+      </Button>
+      <Modal
+        isOpen={isOpen}
+        onClose={onClose}
+        size="xl"
+        scrollBehavior="inside"
+      >
+        <ModalOverlay />
+        <ModalContent>
+          <ModalHeader>
+            WebWand uses the following instructions and tips:
+          </ModalHeader>
+          <ModalCloseButton />
+          <ModalBody>
+            <Notes notes={notes} />
+            <Alert status="info" borderRadius="md" my="1rem">
+              <AlertIcon />
+              <AlertDescription fontSize="1rem">
+                You can customize instructions in the settings menu.
+              </AlertDescription>
+            </Alert>
+          </ModalBody>
+        </ModalContent>
+      </Modal>
+    </>
+  );
+}
diff --git a/src/common/CustomKnowledgeBase/NewKnowledgeForm.tsx b/src/common/CustomKnowledgeBase/NewKnowledgeForm.tsx
index d5cbe0d..11aac7c 100644
--- a/src/common/CustomKnowledgeBase/NewKnowledgeForm.tsx
+++ b/src/common/CustomKnowledgeBase/NewKnowledgeForm.tsx
@@ -9,7 +9,6 @@ import {
   ModalBody,
   ModalFooter,
   Box,
-  Heading,
   InputGroup,
   InputRightElement,
   IconButton,
@@ -235,7 +234,7 @@ const NewKnowledgeForm = ({
   // )
 
   return (
-    <Modal isOpen={isOpen} onClose={closeForm}>
+    <Modal isOpen={isOpen} onClose={closeForm} size="xl">
       <ModalOverlay />
       <ModalContent>
         <ModalCloseButton />
@@ -264,13 +263,13 @@ const NewKnowledgeForm = ({
                 onClose={() => setShowDuplicateAlert(false)}
               />
               <ModalHeader>
-                {isEditMode ? "Edit" : "New"} Host Knowledge
+                {isEditMode ? "Edit" : "New"} Instructions
               </ModalHeader>
               <ModalBody>
                 <FormControl isRequired mb={4}>
-                  <FormLabel>Host</FormLabel>
+                  <FormLabel>Instructions for:</FormLabel>
                   <FormHelperText mb={1} fontSize="xs">
-                    e.g. github.com
+                    host of the website (e.g. google.com)
                   </FormHelperText>
                   <Input
                     name="newHost"
@@ -282,9 +281,6 @@ const NewKnowledgeForm = ({
                 </FormControl>
 
                 {/* Rules Section */}
-                <Heading as="h5" size="sm" mb={2}>
-                  Rules
-                </Heading>
                 {values.rules.map((rule, ruleIndex) => (
                   <Box
                     key={ruleIndex}
@@ -430,19 +426,19 @@ const NewKnowledgeForm = ({
 
                     {/* Notes Section */}
                     <FormControl mb={2}>
-                      <FormLabel>Notes</FormLabel>
+                      <FormLabel>Instructions</FormLabel>
                       {rule.knowledge.notes?.map((note, noteIndex) => (
                         <InputGroup key={noteIndex} size="md" mb={1}>
                           <Field
                             as={Textarea}
                             name={`rules[${ruleIndex}].knowledge.notes[${noteIndex}]`}
-                            placeholder="Enter note"
+                            placeholder="Enter Instructions"
                           />
                           <InputRightElement>
                             {rule.knowledge.notes &&
                               rule.knowledge.notes.length > 1 && (
                                 <IconButton
-                                  aria-label="Remove note"
+                                  aria-label="Remove Instructions"
                                   icon={<SmallCloseIcon />}
                                   variant="ghost"
                                   onClick={() => {
@@ -476,7 +472,7 @@ const NewKnowledgeForm = ({
                         );
                       }}
                     >
-                      Add more notes
+                      Add another instruction
                     </Button>
 
                     {/* Annotation Rules Section */}
@@ -509,7 +505,7 @@ const NewKnowledgeForm = ({
                     setFieldValue("rules", updatedRules);
                   }}
                 >
-                  Add more rules
+                  Add another set of instructions
                 </Button>
               </ModalBody>
               <ModalFooter>
diff --git a/src/common/CustomKnowledgeBase/Notes.tsx b/src/common/CustomKnowledgeBase/Notes.tsx
new file mode 100644
index 0000000..b9f889a
--- /dev/null
+++ b/src/common/CustomKnowledgeBase/Notes.tsx
@@ -0,0 +1,14 @@
+import { UnorderedList, ListItem } from "@chakra-ui/react";
+
+export default function Notes({ notes }: { notes: string[] | undefined }) {
+  if (!notes || notes.length === 0) {
+    return null;
+  }
+  return (
+    <UnorderedList fontSize="0.8rem" styleType="circle">
+      {notes.map((note, index) => (
+        <ListItem key={index}>{note}</ListItem>
+      ))}
+    </UnorderedList>
+  );
+}
diff --git a/src/common/CustomKnowledgeBase/index.tsx b/src/common/CustomKnowledgeBase/index.tsx
index fdb401c..1f49733 100644
--- a/src/common/CustomKnowledgeBase/index.tsx
+++ b/src/common/CustomKnowledgeBase/index.tsx
@@ -1,11 +1,11 @@
 import React, { useState } from "react";
-import { Button, Text, VStack, Box, useDisclosure } from "@chakra-ui/react";
+import { Button, Text, VStack, Box } from "@chakra-ui/react";
 import { useAppState } from "@root/src/state/store";
 import NewKnowledgeForm from "./NewKnowledgeForm";
 import { type EditingData } from "../../helpers/knowledge";
 import DefaultKnowledge from "./DefaultKnowledge";
 import HostKnowledge from "./HostKnowledge";
-import NewKnowledgeJson from "./NewKnowledgeJson";
+// import NewKnowledgeJson from "./NewKnowledgeJson";
 import { findActiveTab } from "../../helpers/browserUtils";
 
 const CustomKnowledgeBase = () => {
@@ -16,11 +16,11 @@ const CustomKnowledgeBase = () => {
   const customKnowledgeBase = useAppState(
     (state) => state.settings.customKnowledgeBase,
   );
-  const {
-    isOpen: isJsonInputOpen,
-    onOpen: openJsonInput,
-    onClose: closeJsonInput,
-  } = useDisclosure();
+  // const {
+  //   isOpen: isJsonInputOpen,
+  //   onOpen: openJsonInput,
+  //   onClose: closeJsonInput,
+  // } = useDisclosure();
   const [defaultHost, setDefaultHost] = useState("");
   const [currentURL, setCurrentUrl] = useState("");
 
@@ -76,10 +76,10 @@ const CustomKnowledgeBase = () => {
           </Box>
         ))
       ) : (
-        <Text>No custom knowledge found</Text>
+        <Text>No instructions found</Text>
       )}
-      <Button onClick={openForm}>Add Host Knowledge with Form</Button>
-      <Button onClick={openJsonInput}>Add Host Knowledge with JSON</Button>
+      <Button onClick={openForm}>Add Instructions</Button>
+      {/* <Button onClick={openJsonInput}>Add Host Knowledge with JSON</Button> */}
       <NewKnowledgeForm
         isOpen={isFormOpen}
         isEditMode={!!editKnowledge}
@@ -88,7 +88,7 @@ const CustomKnowledgeBase = () => {
         defaultHost={defaultHost}
         currentURL={currentURL}
       />
-      <NewKnowledgeJson isOpen={isJsonInputOpen} onClose={closeJsonInput} />
+      {/* <NewKnowledgeJson isOpen={isJsonInputOpen} onClose={closeJsonInput} /> */}
     </VStack>
   );
 };
diff --git a/src/common/Settings.tsx b/src/common/Settings.tsx
index a1abbb6..ef2f451 100644
--- a/src/common/Settings.tsx
+++ b/src/common/Settings.tsx
@@ -109,7 +109,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
           </BreadcrumbItem>
           {view === "knowledge" && (
             <BreadcrumbItem isCurrentPage>
-              <BreadcrumbLink href="#">Knowledge</BreadcrumbLink>
+              <BreadcrumbLink href="#">Instructions</BreadcrumbLink>
             </BreadcrumbItem>
           )}
           {view === "api" && (
@@ -204,7 +204,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
             />
           </Flex>
           <Flex alignItems="center">
-            <FormLabel mb="0">Custom Knowledge Base</FormLabel>
+            <FormLabel mb="0">Custom Instructions</FormLabel>
             <Spacer />
             <Button rightIcon={<EditIcon />} onClick={openCKB}>
               Edit
diff --git a/src/common/TaskUI.tsx b/src/common/TaskUI.tsx
index 5e04acd..5ec7523 100644
--- a/src/common/TaskUI.tsx
+++ b/src/common/TaskUI.tsx
@@ -18,6 +18,7 @@ import TaskHistory from "./TaskHistory";
 import TaskStatus from "./TaskStatus";
 import RecommendedTasks from "./RecommendedTasks";
 import AutosizeTextarea from "./AutosizeTextarea";
+import MatchedNotes from "./CustomKnowledgeBase/MatchedNotes";
 
 function ActionExecutor() {
   const state = useAppState((state) => ({
@@ -138,6 +139,7 @@ const TaskUI = () => {
         <RecommendedTasks runTask={runTaskWithNewInstructions} />
       )}
       {debugMode && <ActionExecutor />}
+      <MatchedNotes />
       <TaskHistory />
     </>
   );
diff --git a/src/state/currentTask.ts b/src/state/currentTask.ts
index a196534..d95cca1 100644
--- a/src/state/currentTask.ts
+++ b/src/state/currentTask.ts
@@ -26,7 +26,7 @@ import { findActiveTab } from "../helpers/browserUtils";
 import { MyStateCreator } from "./store";
 import buildAnnotatedScreenshots from "../helpers/buildAnnotatedScreenshots";
 import { voiceControl } from "../helpers/voiceControl";
-import { fetchKnowledge } from "../helpers/knowledge";
+import { fetchKnowledge, type Knowledge } from "../helpers/knowledge";
 import { hasVisionSupport } from "../helpers/aiSdkUtils";
 
 export type TaskHistoryEntry = {
@@ -39,9 +39,9 @@ export type TaskHistoryEntry = {
 export type CurrentTaskSlice = {
   tabId: number;
   isListening: boolean;
-  instructions: string | null;
   history: TaskHistoryEntry[];
   status: "idle" | "running" | "success" | "error" | "interrupted";
+  knowledgeInUse: Knowledge | null;
   actionStatus:
     | "idle"
     | "attaching-debugger"
@@ -68,10 +68,10 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
 ) => ({
   tabId: -1,
   isListening: false,
-  instructions: null,
   history: [],
   status: "idle",
   actionStatus: "idle",
+  knowledgeInUse: null,
   actions: {
     runTask: async (onError) => {
       const voiceMode = get().settings.voiceMode;
@@ -91,7 +91,6 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
       if (!instructions || get().currentTask.status === "running") return;
 
       set((state) => {
-        state.currentTask.instructions = instructions;
         state.currentTask.history = [];
         state.currentTask.status = "running";
         state.currentTask.actionStatus = "attaching-debugger";
@@ -201,6 +200,10 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
             const url = new URL(activeTab.url ?? "");
             const customKnowledgeBase = get().settings.customKnowledgeBase;
             const knowledge = await fetchKnowledge(url, customKnowledgeBase);
+            set((state) => {
+              state.currentTask.knowledgeInUse = knowledge;
+            });
+
             const [imgData, labelData] = await buildAnnotatedScreenshots(
               tabId,
               knowledge,