From cda754c1066490d817056daafe598d12f690cd1e Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sat, 25 Jan 2025 17:17:20 +0100
Subject: [PATCH 01/15] feat: add hitl state

---
 src/state/hitl.ts  | 22 ++++++++++++++++++++++
 src/state/store.ts |  3 +++
 2 files changed, 25 insertions(+)
 create mode 100644 src/state/hitl.ts

diff --git a/src/state/hitl.ts b/src/state/hitl.ts
new file mode 100644
index 0000000..53f997c
--- /dev/null
+++ b/src/state/hitl.ts
@@ -0,0 +1,22 @@
+import { MyStateCreator } from "./store";
+import { Action } from "../helpers/vision-agent/parseResponse";
+
+export type HitlSlice = {
+  proposedAction: Action | null;
+  setProposedAction: (action: Action | null) => void;
+  userDecision: "approve" | "reject" | null;
+  setUserDecision: (decision: "approve" | "reject" | null) => void;
+};
+
+export const createHitlSlice: MyStateCreator<HitlSlice> = (set) => ({
+  proposedAction: null,
+  setProposedAction: (action) =>
+    set((state) => {
+      state.hitl.proposedAction = action;
+    }),
+  userDecision: null,
+  setUserDecision: (decision) =>
+    set((state) => {
+      state.hitl.userDecision = decision;
+    }),
+});
diff --git a/src/state/store.ts b/src/state/store.ts
index e317987..64fc67b 100644
--- a/src/state/store.ts
+++ b/src/state/store.ts
@@ -5,12 +5,14 @@ import { createJSONStorage, devtools, persist } from "zustand/middleware";
 import { createCurrentTaskSlice, CurrentTaskSlice } from "./currentTask";
 import { createUiSlice, UiSlice } from "./ui";
 import { createSettingsSlice, SettingsSlice } from "./settings";
+import { createHitlSlice, HitlSlice } from "./hitl";
 import { findBestMatchingModel } from "../helpers/aiSdkUtils";
 
 export type StoreType = {
   currentTask: CurrentTaskSlice;
   ui: UiSlice;
   settings: SettingsSlice;
+  hitl: HitlSlice;
 };
 
 export type MyStateCreator<T> = StateCreator<
@@ -27,6 +29,7 @@ export const useAppState = create<StoreType>()(
         currentTask: createCurrentTaskSlice(...a),
         ui: createUiSlice(...a),
         settings: createSettingsSlice(...a),
+        hitl: createHitlSlice(...a),
       })),
     ),
     {

From 7f164c5a305d341de9a1db0fbd6b3d9af393385b Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sat, 25 Jan 2025 17:44:21 +0100
Subject: [PATCH 02/15] feat: make 'safety checkpoints' button in settings

---
 src/common/Settings.tsx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/common/Settings.tsx b/src/common/Settings.tsx
index fbc0aa7..94964e8 100644
--- a/src/common/Settings.tsx
+++ b/src/common/Settings.tsx
@@ -35,7 +35,7 @@ type SettingsProps = {
 };
 
 const Settings = ({ setInSettingsView }: SettingsProps) => {
-  const [view, setView] = useState<"settings" | "knowledge" | "api">(
+  const [view, setView] = useState<"settings" | "knowledge" | "api" | "hitl">(
     "settings",
   );
   const state = useAppState((state) => ({
@@ -118,6 +118,11 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
               <BreadcrumbLink href="#">API</BreadcrumbLink>
             </BreadcrumbItem>
           )}
+          {view === "hitl" && (
+            <BreadcrumbItem isCurrentPage>
+              <BreadcrumbLink href="#">Safety Checkpoints</BreadcrumbLink>
+            </BreadcrumbItem>
+          )}
         </Breadcrumb>
       </HStack>
       {view === "knowledge" && <CustomKnowledgeBase />}
@@ -225,6 +230,18 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
               Edit
             </Button>
           </Flex>
+          <Flex alignItems="center">
+            <Box>
+              <FormLabel mb="0">Safety Checkpoints</FormLabel>
+              <FormHelperText>
+                Define actions that require your approval
+              </FormHelperText>
+            </Box>
+            <Spacer />
+            <Button rightIcon={<EditIcon />} onClick={() => setView("hitl")}>
+              Edit
+            </Button>
+          </Flex>
         </FormControl>
       )}
     </>

From a93f59bf056c4ccdbcf9ba571ec7fc05aa0d51b6 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sat, 25 Jan 2025 18:50:38 +0100
Subject: [PATCH 03/15] feat: add HITL settings form and integration in
 settings

---
 src/common/HITLSettings/NewHITLForm.tsx | 63 ++++++++++++++++++++++++
 src/common/HITLSettings/index.tsx       | 64 +++++++++++++++++++++++++
 src/common/Settings.tsx                 |  7 ++-
 3 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 src/common/HITLSettings/NewHITLForm.tsx
 create mode 100644 src/common/HITLSettings/index.tsx

diff --git a/src/common/HITLSettings/NewHITLForm.tsx b/src/common/HITLSettings/NewHITLForm.tsx
new file mode 100644
index 0000000..f097847
--- /dev/null
+++ b/src/common/HITLSettings/NewHITLForm.tsx
@@ -0,0 +1,63 @@
+import React from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalBody,
+  ModalCloseButton,
+  FormControl,
+  FormLabel,
+  Input,
+  Textarea,
+  Button,
+  VStack,
+} from "@chakra-ui/react";
+
+type NewHITLFormProps = {
+  isOpen: boolean;
+  isEditMode: boolean;
+  editRule?: {
+    id: string;
+    pattern: string;
+    description: string;
+  };
+  closeForm: () => void;
+};
+
+const NewHITLForm: React.FC<NewHITLFormProps> = ({
+  isOpen,
+  isEditMode,
+  // editRule, // Unused for now
+  closeForm,
+}) => {
+  // TODO: Implement form logic
+  return (
+    <Modal isOpen={isOpen} onClose={closeForm} size="xl">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>
+          {isEditMode ? "Edit Safety Checkpoint" : "Add Safety Checkpoint"}
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <VStack spacing={4} pb={4}>
+            <FormControl isRequired>
+              <FormLabel>Description</FormLabel>
+              <Input placeholder="e.g., Confirm before posting to Twitter" />
+            </FormControl>
+            <FormControl isRequired>
+              <FormLabel>Pattern</FormLabel>
+              <Textarea placeholder="e.g., Before clicking 'Post' button" />
+            </FormControl>
+            <Button colorScheme="blue" w="full">
+              {isEditMode ? "Update" : "Add"} Checkpoint
+            </Button>
+          </VStack>
+        </ModalBody>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+export default NewHITLForm;
diff --git a/src/common/HITLSettings/index.tsx b/src/common/HITLSettings/index.tsx
new file mode 100644
index 0000000..a38c09b
--- /dev/null
+++ b/src/common/HITLSettings/index.tsx
@@ -0,0 +1,64 @@
+import React, { useState } from "react";
+import { Button, Text, VStack, Box, Alert, AlertIcon } from "@chakra-ui/react";
+// import { useAppState } from "../../state/store"; // Unused for now
+import NewHITLForm from "./NewHITLForm";
+
+// TODO: Move to types file when implementing
+type HITLRule = {
+  id: string;
+  pattern: string;
+  description: string;
+};
+
+const HITLSettings = () => {
+  const [isFormOpen, setIsFormOpen] = useState(false);
+  const [editRule, setEditRule] = useState<HITLRule | undefined>(undefined);
+
+  // TODO: Implement in store
+  const hitlRules: HITLRule[] = [];
+
+  const closeForm = () => {
+    setEditRule(undefined);
+    setIsFormOpen(false);
+  };
+
+  // const openEditForm = (rule: HITLRule) => { // Unused for now
+  //   setEditRule(rule);
+  //   setIsFormOpen(true);
+  // }
+
+  return (
+    <VStack spacing={4}>
+      <Alert status="info" borderRadius="md">
+        <AlertIcon />
+        <Text>
+          Add checkpoints to make Fuji ask for your permission before performing
+          certain actions.
+        </Text>
+      </Alert>
+      {hitlRules.length > 0 ? (
+        hitlRules.map((rule) => (
+          <Box key={rule.id} w="full" p={4} borderWidth="1px" borderRadius="lg">
+            <Text fontWeight="bold">{rule.description}</Text>
+            <Text fontSize="sm" color="gray.600">
+              Pattern: {rule.pattern}
+            </Text>
+          </Box>
+        ))
+      ) : (
+        <Text>No safety checkpoints configured</Text>
+      )}
+      <Button onClick={() => setIsFormOpen(true)}>Add Safety Checkpoint</Button>
+      {isFormOpen && (
+        <NewHITLForm
+          isOpen={isFormOpen}
+          isEditMode={!!editRule}
+          editRule={editRule}
+          closeForm={closeForm}
+        />
+      )}
+    </VStack>
+  );
+};
+
+export default HITLSettings;
diff --git a/src/common/Settings.tsx b/src/common/Settings.tsx
index 94964e8..01f7898 100644
--- a/src/common/Settings.tsx
+++ b/src/common/Settings.tsx
@@ -26,6 +26,7 @@ import ModelDropdown from "./settings/ModelDropdown";
 import AgentModeDropdown from "./settings/AgentModeDropdown";
 import { callRPC } from "../helpers/rpc/pageRPC";
 import CustomKnowledgeBase from "./CustomKnowledgeBase";
+import HITLSettings from "./HITLSettings";
 import SetAPIKey from "./settings/SetAPIKey";
 import { debugMode } from "../constants";
 import { isValidModelSettings } from "../helpers/aiSdkUtils";
@@ -53,6 +54,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
 
   const closeSetting = () => setInSettingsView(false);
   const openCKB = () => setView("knowledge");
+  const openHITL = () => setView("hitl");
   const backToSettings = () => setView("settings");
 
   async function checkMicrophonePermission(): Promise<PermissionState> {
@@ -120,7 +122,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
           )}
           {view === "hitl" && (
             <BreadcrumbItem isCurrentPage>
-              <BreadcrumbLink href="#">Safety Checkpoints</BreadcrumbLink>
+              <BreadcrumbLink href="#">Checkpoints</BreadcrumbLink>
             </BreadcrumbItem>
           )}
         </Breadcrumb>
@@ -134,6 +136,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
           onClose={backToSettings}
         />
       )}
+      {view === "hitl" && <HITLSettings />}
       {view === "settings" && (
         <FormControl
           as={VStack}
@@ -238,7 +241,7 @@ const Settings = ({ setInSettingsView }: SettingsProps) => {
               </FormHelperText>
             </Box>
             <Spacer />
-            <Button rightIcon={<EditIcon />} onClick={() => setView("hitl")}>
+            <Button rightIcon={<EditIcon />} onClick={openHITL}>
               Edit
             </Button>
           </Flex>

From c11ca1e47e9daaa3086b6a5c46736a00a2e96ec3 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sun, 26 Jan 2025 11:05:53 +0100
Subject: [PATCH 04/15] feat: save hitlrules v1

---
 src/common/HITLSettings/NewHITLForm.tsx | 31 +++++++++++++----
 src/common/HITLSettings/index.tsx       | 44 ++++++++++++++++++-------
 src/helpers/knowledge/index.ts          |  6 ++++
 src/state/settings.ts                   |  4 ++-
 src/state/store.ts                      |  2 +-
 5 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/src/common/HITLSettings/NewHITLForm.tsx b/src/common/HITLSettings/NewHITLForm.tsx
index f097847..13ef06e 100644
--- a/src/common/HITLSettings/NewHITLForm.tsx
+++ b/src/common/HITLSettings/NewHITLForm.tsx
@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useState } from "react";
 import {
   Modal,
   ModalOverlay,
@@ -23,15 +23,26 @@ type NewHITLFormProps = {
     description: string;
   };
   closeForm: () => void;
+  onSave: (rule: { pattern: string; description: string }) => void;
 };
 
 const NewHITLForm: React.FC<NewHITLFormProps> = ({
   isOpen,
   isEditMode,
-  // editRule, // Unused for now
+  editRule,
   closeForm,
+  onSave,
 }) => {
-  // TODO: Implement form logic
+  const [pattern, setPattern] = useState(editRule?.pattern || "");
+  const [description, setDescription] = useState(editRule?.description || "");
+
+  const handleSubmit = () => {
+    if (!pattern || !description) return;
+    onSave({ pattern, description });
+    setPattern("");
+    setDescription("");
+  };
+
   return (
     <Modal isOpen={isOpen} onClose={closeForm} size="xl">
       <ModalOverlay />
@@ -44,13 +55,21 @@ const NewHITLForm: React.FC<NewHITLFormProps> = ({
           <VStack spacing={4} pb={4}>
             <FormControl isRequired>
               <FormLabel>Description</FormLabel>
-              <Input placeholder="e.g., Confirm before posting to Twitter" />
+              <Input
+                value={description}
+                onChange={(e) => setDescription(e.target.value)}
+                placeholder="e.g., Confirm before posting to Facebook"
+              />
             </FormControl>
             <FormControl isRequired>
               <FormLabel>Pattern</FormLabel>
-              <Textarea placeholder="e.g., Before clicking 'Post' button" />
+              <Textarea
+                value={pattern}
+                onChange={(e) => setPattern(e.target.value)}
+                placeholder="e.g., Confirm before clicking 'Post' button on facebook.com"
+              />
             </FormControl>
-            <Button colorScheme="blue" w="full">
+            <Button colorScheme="blue" w="full" onClick={handleSubmit}>
               {isEditMode ? "Update" : "Add"} Checkpoint
             </Button>
           </VStack>
diff --git a/src/common/HITLSettings/index.tsx b/src/common/HITLSettings/index.tsx
index a38c09b..79f4450 100644
--- a/src/common/HITLSettings/index.tsx
+++ b/src/common/HITLSettings/index.tsx
@@ -1,31 +1,50 @@
 import React, { useState } from "react";
 import { Button, Text, VStack, Box, Alert, AlertIcon } from "@chakra-ui/react";
-// import { useAppState } from "../../state/store"; // Unused for now
+import { useAppState } from "../../state/store";
 import NewHITLForm from "./NewHITLForm";
-
-// TODO: Move to types file when implementing
-type HITLRule = {
-  id: string;
-  pattern: string;
-  description: string;
-};
+import type { HITLRule } from "../../helpers/knowledge";
 
 const HITLSettings = () => {
   const [isFormOpen, setIsFormOpen] = useState(false);
   const [editRule, setEditRule] = useState<HITLRule | undefined>(undefined);
 
-  // TODO: Implement in store
-  const hitlRules: HITLRule[] = [];
+  const { hitlRules, updateSettings } = useAppState((state) => ({
+    hitlRules: state.settings.hitlRules,
+    updateSettings: state.settings.actions.update,
+  }));
 
   const closeForm = () => {
     setEditRule(undefined);
     setIsFormOpen(false);
   };
 
-  // const openEditForm = (rule: HITLRule) => { // Unused for now
+  const handleSaveRule = (rule: Omit<HITLRule, "id">) => {
+    if (editRule) {
+      // Update existing rule
+      const updatedRules = hitlRules.map((r) =>
+        r.id === editRule.id ? { ...rule, id: editRule.id } : r,
+      );
+      updateSettings({ hitlRules: updatedRules });
+    } else {
+      // Add new rule
+      const newRule = {
+        ...rule,
+        id: crypto.randomUUID(),
+      };
+      updateSettings({ hitlRules: [...hitlRules, newRule] });
+    }
+    closeForm();
+  };
+
+  // const handleDeleteRule = (id: string) => {
+  //   const updatedRules = hitlRules.filter((rule) => rule.id !== id);
+  //   updateSettings({ hitlRules: updatedRules });
+  // };
+
+  // const openEditForm = (rule: HITLRule) => {
   //   setEditRule(rule);
   //   setIsFormOpen(true);
-  // }
+  // };
 
   return (
     <VStack spacing={4}>
@@ -55,6 +74,7 @@ const HITLSettings = () => {
           isEditMode={!!editRule}
           editRule={editRule}
           closeForm={closeForm}
+          onSave={handleSaveRule}
         />
       )}
     </VStack>
diff --git a/src/helpers/knowledge/index.ts b/src/helpers/knowledge/index.ts
index b12fdce..5fb38ec 100644
--- a/src/helpers/knowledge/index.ts
+++ b/src/helpers/knowledge/index.ts
@@ -45,6 +45,12 @@ export type LocationInfo = {
   pathname: string;
 };
 
+export type HITLRule = {
+  id: string;
+  pattern: string;
+  description: string;
+};
+
 export function fetchKnowledge(
   location: LocationInfo,
   customKnowledgeBase?: Data,
diff --git a/src/state/settings.ts b/src/state/settings.ts
index 4083dbe..4fcbcf4 100644
--- a/src/state/settings.ts
+++ b/src/state/settings.ts
@@ -1,4 +1,4 @@
-import { type Data } from "../helpers/knowledge/index";
+import { type Data, type HITLRule } from "../helpers/knowledge/index";
 import { MyStateCreator } from "./store";
 import {
   SupportedModels,
@@ -16,6 +16,7 @@ export type SettingsSlice = {
   agentMode: AgentMode;
   voiceMode: boolean;
   customKnowledgeBase: Data;
+  hitlRules: HITLRule[];
   actions: {
     update: (values: Partial<SettingsSlice>) => void;
   };
@@ -30,6 +31,7 @@ export const createSettingsSlice: MyStateCreator<SettingsSlice> = (set) => ({
   selectedModel: SupportedModels.Gpt4Turbo,
   voiceMode: false,
   customKnowledgeBase: {},
+  hitlRules: [],
   actions: {
     update: (values) => {
       set((state) => {
diff --git a/src/state/store.ts b/src/state/store.ts
index 64fc67b..0b0e639 100644
--- a/src/state/store.ts
+++ b/src/state/store.ts
@@ -36,7 +36,6 @@ export const useAppState = create<StoreType>()(
       name: "app-state",
       storage: createJSONStorage(() => localStorage),
       partialize: (state) => ({
-        // Stuff we want to persist
         ui: {
           instructions: state.ui.instructions,
         },
@@ -50,6 +49,7 @@ export const useAppState = create<StoreType>()(
           selectedModel: state.settings.selectedModel,
           voiceMode: state.settings.voiceMode,
           customKnowledgeBase: state.settings.customKnowledgeBase,
+          hitlRules: state.settings.hitlRules,
         },
       }),
       merge: (persistedState, currentState) => {

From 6635b6e52d24168c3090880efc57dbfcf0f5f5ec Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sun, 26 Jan 2025 11:31:42 +0100
Subject: [PATCH 05/15] feat: hitlrules v2 edit and delete

---
 src/common/HITLSettings/CheckpointItem.tsx | 46 ++++++++++++++++++++++
 src/common/HITLSettings/index.tsx          | 36 +++++++++--------
 src/helpers/hitl/index.ts                  |  5 +++
 3 files changed, 70 insertions(+), 17 deletions(-)
 create mode 100644 src/common/HITLSettings/CheckpointItem.tsx
 create mode 100644 src/helpers/hitl/index.ts

diff --git a/src/common/HITLSettings/CheckpointItem.tsx b/src/common/HITLSettings/CheckpointItem.tsx
new file mode 100644
index 0000000..42002bc
--- /dev/null
+++ b/src/common/HITLSettings/CheckpointItem.tsx
@@ -0,0 +1,46 @@
+import { DeleteIcon, EditIcon } from "@chakra-ui/icons";
+import { Box, Text, Flex, IconButton, Tooltip } from "@chakra-ui/react";
+import type { HITLRule } from "../../helpers/knowledge";
+
+type CheckpointItemProps = {
+  rule: HITLRule;
+  onEdit: (rule: HITLRule) => void;
+  onDelete: (id: string) => void;
+};
+
+const CheckpointItem = ({ rule, onEdit, onDelete }: CheckpointItemProps) => {
+  return (
+    <Box w="full" p={4} borderWidth="1px" borderRadius="lg">
+      <Flex alignItems="flex-start">
+        <Box flex="1">
+          <Text fontWeight="bold">{rule.description}</Text>
+          <Text fontSize="sm" color="gray.600">
+            Pattern: {rule.pattern}
+          </Text>
+        </Box>
+        <Box>
+          <Tooltip label="Edit checkpoint">
+            <IconButton
+              aria-label="Edit checkpoint"
+              icon={<EditIcon />}
+              size="sm"
+              variant="ghost"
+              onClick={() => onEdit(rule)}
+            />
+          </Tooltip>
+          <Tooltip label="Remove checkpoint">
+            <IconButton
+              aria-label="Remove checkpoint"
+              icon={<DeleteIcon />}
+              size="sm"
+              variant="ghost"
+              onClick={() => onDelete(rule.id)}
+            />
+          </Tooltip>
+        </Box>
+      </Flex>
+    </Box>
+  );
+};
+
+export default CheckpointItem;
diff --git a/src/common/HITLSettings/index.tsx b/src/common/HITLSettings/index.tsx
index 79f4450..f433f9e 100644
--- a/src/common/HITLSettings/index.tsx
+++ b/src/common/HITLSettings/index.tsx
@@ -1,7 +1,8 @@
 import React, { useState } from "react";
-import { Button, Text, VStack, Box, Alert, AlertIcon } from "@chakra-ui/react";
+import { Button, Text, VStack, Alert, AlertIcon } from "@chakra-ui/react";
 import { useAppState } from "../../state/store";
 import NewHITLForm from "./NewHITLForm";
+import CheckpointItem from "./CheckpointItem";
 import type { HITLRule } from "../../helpers/knowledge";
 
 const HITLSettings = () => {
@@ -23,11 +24,11 @@ const HITLSettings = () => {
       // Update existing rule
       const updatedRules = hitlRules.map((r) =>
         r.id === editRule.id ? { ...rule, id: editRule.id } : r,
-      );
+      ) as HITLRule[];
       updateSettings({ hitlRules: updatedRules });
     } else {
       // Add new rule
-      const newRule = {
+      const newRule: HITLRule = {
         ...rule,
         id: crypto.randomUUID(),
       };
@@ -36,33 +37,34 @@ const HITLSettings = () => {
     closeForm();
   };
 
-  // const handleDeleteRule = (id: string) => {
-  //   const updatedRules = hitlRules.filter((rule) => rule.id !== id);
-  //   updateSettings({ hitlRules: updatedRules });
-  // };
+  const handleDeleteRule = (id: string) => {
+    const updatedRules = hitlRules.filter((rule) => rule.id !== id);
+    updateSettings({ hitlRules: updatedRules });
+  };
 
-  // const openEditForm = (rule: HITLRule) => {
-  //   setEditRule(rule);
-  //   setIsFormOpen(true);
-  // };
+  const openEditForm = (rule: HITLRule) => {
+    setEditRule(rule);
+    setIsFormOpen(true);
+  };
 
   return (
     <VStack spacing={4}>
       <Alert status="info" borderRadius="md">
         <AlertIcon />
         <Text>
+          {" "}
           Add checkpoints to make Fuji ask for your permission before performing
           certain actions.
         </Text>
       </Alert>
       {hitlRules.length > 0 ? (
         hitlRules.map((rule) => (
-          <Box key={rule.id} w="full" p={4} borderWidth="1px" borderRadius="lg">
-            <Text fontWeight="bold">{rule.description}</Text>
-            <Text fontSize="sm" color="gray.600">
-              Pattern: {rule.pattern}
-            </Text>
-          </Box>
+          <CheckpointItem
+            key={rule.id}
+            rule={rule}
+            onEdit={openEditForm}
+            onDelete={handleDeleteRule}
+          />
         ))
       ) : (
         <Text>No safety checkpoints configured</Text>
diff --git a/src/helpers/hitl/index.ts b/src/helpers/hitl/index.ts
new file mode 100644
index 0000000..fe0c85b
--- /dev/null
+++ b/src/helpers/hitl/index.ts
@@ -0,0 +1,5 @@
+export type CheckpointRule = {
+  id: string;
+  pattern: string;
+  description: string;
+};

From c97a1db0079b8a56ee6bd39784db4e7d7459bfd9 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sun, 26 Jan 2025 11:35:54 +0100
Subject: [PATCH 06/15] refactor: change hitlrule to checkpointrule

---
 src/common/HITLSettings/CheckpointItem.tsx |  6 +++---
 src/common/HITLSettings/index.tsx          | 14 ++++++++------
 src/helpers/knowledge/index.ts             |  6 ------
 src/state/settings.ts                      |  5 +++--
 4 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/src/common/HITLSettings/CheckpointItem.tsx b/src/common/HITLSettings/CheckpointItem.tsx
index 42002bc..21ea04f 100644
--- a/src/common/HITLSettings/CheckpointItem.tsx
+++ b/src/common/HITLSettings/CheckpointItem.tsx
@@ -1,10 +1,10 @@
 import { DeleteIcon, EditIcon } from "@chakra-ui/icons";
 import { Box, Text, Flex, IconButton, Tooltip } from "@chakra-ui/react";
-import type { HITLRule } from "../../helpers/knowledge";
+import type { CheckpointRule } from "../../helpers/hitl";
 
 type CheckpointItemProps = {
-  rule: HITLRule;
-  onEdit: (rule: HITLRule) => void;
+  rule: CheckpointRule;
+  onEdit: (rule: CheckpointRule) => void;
   onDelete: (id: string) => void;
 };
 
diff --git a/src/common/HITLSettings/index.tsx b/src/common/HITLSettings/index.tsx
index f433f9e..3be0b2d 100644
--- a/src/common/HITLSettings/index.tsx
+++ b/src/common/HITLSettings/index.tsx
@@ -3,11 +3,13 @@ import { Button, Text, VStack, Alert, AlertIcon } from "@chakra-ui/react";
 import { useAppState } from "../../state/store";
 import NewHITLForm from "./NewHITLForm";
 import CheckpointItem from "./CheckpointItem";
-import type { HITLRule } from "../../helpers/knowledge";
+import type { CheckpointRule } from "../../helpers/hitl";
 
 const HITLSettings = () => {
   const [isFormOpen, setIsFormOpen] = useState(false);
-  const [editRule, setEditRule] = useState<HITLRule | undefined>(undefined);
+  const [editRule, setEditRule] = useState<CheckpointRule | undefined>(
+    undefined,
+  );
 
   const { hitlRules, updateSettings } = useAppState((state) => ({
     hitlRules: state.settings.hitlRules,
@@ -19,16 +21,16 @@ const HITLSettings = () => {
     setIsFormOpen(false);
   };
 
-  const handleSaveRule = (rule: Omit<HITLRule, "id">) => {
+  const handleSaveRule = (rule: Omit<CheckpointRule, "id">) => {
     if (editRule) {
       // Update existing rule
       const updatedRules = hitlRules.map((r) =>
         r.id === editRule.id ? { ...rule, id: editRule.id } : r,
-      ) as HITLRule[];
+      ) as CheckpointRule[];
       updateSettings({ hitlRules: updatedRules });
     } else {
       // Add new rule
-      const newRule: HITLRule = {
+      const newRule: CheckpointRule = {
         ...rule,
         id: crypto.randomUUID(),
       };
@@ -42,7 +44,7 @@ const HITLSettings = () => {
     updateSettings({ hitlRules: updatedRules });
   };
 
-  const openEditForm = (rule: HITLRule) => {
+  const openEditForm = (rule: CheckpointRule) => {
     setEditRule(rule);
     setIsFormOpen(true);
   };
diff --git a/src/helpers/knowledge/index.ts b/src/helpers/knowledge/index.ts
index 5fb38ec..b12fdce 100644
--- a/src/helpers/knowledge/index.ts
+++ b/src/helpers/knowledge/index.ts
@@ -45,12 +45,6 @@ export type LocationInfo = {
   pathname: string;
 };
 
-export type HITLRule = {
-  id: string;
-  pattern: string;
-  description: string;
-};
-
 export function fetchKnowledge(
   location: LocationInfo,
   customKnowledgeBase?: Data,
diff --git a/src/state/settings.ts b/src/state/settings.ts
index 4fcbcf4..a01d3ec 100644
--- a/src/state/settings.ts
+++ b/src/state/settings.ts
@@ -1,4 +1,5 @@
-import { type Data, type HITLRule } from "../helpers/knowledge/index";
+import { type Data } from "../helpers/knowledge";
+import { type CheckpointRule } from "../helpers/hitl";
 import { MyStateCreator } from "./store";
 import {
   SupportedModels,
@@ -16,7 +17,7 @@ export type SettingsSlice = {
   agentMode: AgentMode;
   voiceMode: boolean;
   customKnowledgeBase: Data;
-  hitlRules: HITLRule[];
+  hitlRules: CheckpointRule[];
   actions: {
     update: (values: Partial<SettingsSlice>) => void;
   };

From ffcd1d1e1257377e6d119f692455a8239e257188 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sun, 26 Jan 2025 12:19:02 +0100
Subject: [PATCH 07/15] feat: add pending-approval state

---
 src/state/hitl.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/state/hitl.ts b/src/state/hitl.ts
index 53f997c..61a80d1 100644
--- a/src/state/hitl.ts
+++ b/src/state/hitl.ts
@@ -6,6 +6,8 @@ export type HitlSlice = {
   setProposedAction: (action: Action | null) => void;
   userDecision: "approve" | "reject" | null;
   setUserDecision: (decision: "approve" | "reject" | null) => void;
+  isPendingApproval: boolean;
+  setIsPendingApproval: (isPending: boolean) => void;
 };
 
 export const createHitlSlice: MyStateCreator<HitlSlice> = (set) => ({
@@ -19,4 +21,9 @@ export const createHitlSlice: MyStateCreator<HitlSlice> = (set) => ({
     set((state) => {
       state.hitl.userDecision = decision;
     }),
+  isPendingApproval: false,
+  setIsPendingApproval: (isPending) =>
+    set((state) => {
+      state.hitl.isPendingApproval = isPending;
+    }),
 });

From 437099e7a097ea34c1af379ffc9c3074c2860a30 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Sun, 26 Jan 2025 13:37:27 +0100
Subject: [PATCH 08/15] feat: add draft approvaldialog

---
 src/common/App.tsx            |  2 ++
 src/common/ApprovalDialog.tsx | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 src/common/ApprovalDialog.tsx

diff --git a/src/common/App.tsx b/src/common/App.tsx
index 24d2e33..555dc50 100644
--- a/src/common/App.tsx
+++ b/src/common/App.tsx
@@ -14,6 +14,7 @@ import { useAppState } from "../state/store";
 import SetAPIKey from "./settings/SetAPIKey";
 import TaskUI from "./TaskUI";
 import Settings from "./Settings";
+import ApprovalDialog from "./ApprovalDialog";
 
 const App = () => {
   const hasAPIKey = useAppState(
@@ -23,6 +24,7 @@ const App = () => {
 
   return (
     <ChakraProvider>
+      <ApprovalDialog />
       <Box p="8" pb="24" fontSize="lg" w="full">
         <HStack mb={4} alignItems="center">
           <Heading as="h1" size="lg" flex={1}>
diff --git a/src/common/ApprovalDialog.tsx b/src/common/ApprovalDialog.tsx
new file mode 100644
index 0000000..25fbf4a
--- /dev/null
+++ b/src/common/ApprovalDialog.tsx
@@ -0,0 +1,34 @@
+import { useAppState } from "../state/store";
+
+const ApprovalDialog = () => {
+  const { isPendingApproval, setUserDecision } = useAppState(
+    (state) => state.hitl,
+  );
+
+  if (!isPendingApproval) return null;
+
+  return (
+    <div className="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
+      <div className="bg-white p-6 rounded-lg shadow-lg max-w-md">
+        <h2 className="text-lg font-bold mb-4">Approval Required</h2>
+        <p className="mb-4">Pending Approval Box Placeholder</p>
+        <div className="flex justify-end gap-2">
+          <button
+            className="px-4 py-2 bg-red-500 text-white rounded"
+            onClick={() => setUserDecision("reject")}
+          >
+            Reject
+          </button>
+          <button
+            className="px-4 py-2 bg-green-500 text-white rounded"
+            onClick={() => setUserDecision("approve")}
+          >
+            Approve
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default ApprovalDialog;

From c15279ed2865c99d4f31106e7714bbc15aaab563 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 19:29:26 +0100
Subject: [PATCH 09/15] feat: add method to check if thought matches hitl rule

---
 src/helpers/hitl/index.ts | 67 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/src/helpers/hitl/index.ts b/src/helpers/hitl/index.ts
index fe0c85b..026730a 100644
--- a/src/helpers/hitl/index.ts
+++ b/src/helpers/hitl/index.ts
@@ -1,5 +1,72 @@
+import { useAppState } from "../../state/store";
+import { fetchResponseFromModel } from "../aiSdkUtils";
+import { QueryResult } from "../vision-agent/determineNextAction";
+
+const systemMessage = `
+You are an oversight system for a browser automation assistant.
+
+You will receive a JSON with two fields:
+  hitl_rule: a rule that describes what kind of actions require human approval before execution. These actions trigger a safety checkpoint.
+  action: a JSON describing an action that the assistant intends to execute.
+
+You will be asked to determine if the assistant's intended action falls within the scope of actions described by the rule.
+If they do, then they require human approval and you should respond with "true".
+If they do not, then they do not require human approval and you should respond with "false".
+
+You response will be strictly "true" or "false". It will not contain any further text, including any additional context or explanation.
+`;
+
 export type CheckpointRule = {
   id: string;
   pattern: string;
   description: string;
 };
+
+export function hasCheckpointRules(): boolean {
+  const store = useAppState.getState();
+  return store.settings.hitlRules.length > 0;
+}
+
+// TODO: remove console logs
+export async function matchesCheckpointRule(
+  query: QueryResult,
+): Promise<boolean> {
+  // If no rules defined, don't require approval
+  if (!hasCheckpointRules()) {
+    return false;
+  }
+  console.log("Checking checkpoint rules...");
+  const store = useAppState.getState();
+  const rules = store.settings.hitlRules;
+  console.log("Rules: ", rules);
+
+  const rawResponse = query?.rawResponse;
+
+  for (const rule of rules) {
+    const prompt = JSON.stringify(
+      {
+        hitl_rule: rule.description,
+        action: rawResponse,
+      },
+      null,
+      2,
+    );
+    console.log("Prompt: ", prompt);
+    const model = useAppState.getState().settings.selectedModel;
+
+    const completion = await fetchResponseFromModel(model, {
+      systemMessage: systemMessage,
+      prompt,
+      jsonMode: false,
+    });
+    console.log("Completion: ", completion);
+
+    const needsApproval = completion.rawResponse.toLowerCase() === "true";
+    if (needsApproval) {
+      return true;
+    }
+  }
+
+  // If no rules matched, don't require approval
+  return false;
+}

From b071d20a6ecc4770f6c721532f69eb6e8775dbfc Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 19:50:09 +0100
Subject: [PATCH 10/15] feat: triggering rule interrupts contol flow

---
 src/state/currentTask.ts | 27 +++++++++++++++++++++++++++
 src/state/hitl.ts        | 16 +++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/src/state/currentTask.ts b/src/state/currentTask.ts
index 1e852d3..85e3f09 100644
--- a/src/state/currentTask.ts
+++ b/src/state/currentTask.ts
@@ -28,6 +28,7 @@ import buildAnnotatedScreenshots from "../helpers/buildAnnotatedScreenshots";
 import { voiceControl } from "../helpers/voiceControl";
 import { fetchKnowledge, type Knowledge } from "../helpers/knowledge";
 import { isValidModelSettings, AgentMode } from "../helpers/aiSdkUtils";
+import { matchesCheckpointRule } from "../helpers/hitl";
 
 export type TaskHistoryEntry = {
   prompt: string;
@@ -268,6 +269,32 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
 
           if (wasStopped()) break;
 
+          const needsApproval = await matchesCheckpointRule(query);
+
+          if (needsApproval && query) {
+            set((state) => {
+              state.hitl.isPendingApproval = true;
+              state.hitl.proposedAction = query.action;
+              state.hitl.userDecision = null;
+            });
+
+            const decision = await get().hitl.waitForApproval();
+
+            // Reset HITL state
+            set((state) => {
+              state.hitl.isPendingApproval = false;
+              state.hitl.proposedAction = null;
+              state.hitl.userDecision = null;
+            });
+
+            if (decision === "reject") {
+              set((state) => {
+                state.currentTask.status = "interrupted";
+              });
+              break;
+            }
+          }
+
           const shouldContinue = await performAction(query);
 
           if (wasStopped() || !shouldContinue) break;
diff --git a/src/state/hitl.ts b/src/state/hitl.ts
index 61a80d1..2e12b46 100644
--- a/src/state/hitl.ts
+++ b/src/state/hitl.ts
@@ -8,9 +8,10 @@ export type HitlSlice = {
   setUserDecision: (decision: "approve" | "reject" | null) => void;
   isPendingApproval: boolean;
   setIsPendingApproval: (isPending: boolean) => void;
+  waitForApproval: () => Promise<"approve" | "reject">;
 };
 
-export const createHitlSlice: MyStateCreator<HitlSlice> = (set) => ({
+export const createHitlSlice: MyStateCreator<HitlSlice> = (set, get) => ({
   proposedAction: null,
   setProposedAction: (action) =>
     set((state) => {
@@ -26,4 +27,17 @@ export const createHitlSlice: MyStateCreator<HitlSlice> = (set) => ({
     set((state) => {
       state.hitl.isPendingApproval = isPending;
     }),
+  waitForApproval: async () => {
+    return new Promise((resolve) => {
+      const checkDecision = () => {
+        const decision = get().hitl.userDecision;
+        if (decision) {
+          resolve(decision);
+        } else {
+          setTimeout(checkDecision, 100);
+        }
+      };
+      checkDecision();
+    });
+  },
 });

From fff2bfd6d787cea95b4699eb06fbdf647fa5d8cb Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 20:08:28 +0100
Subject: [PATCH 11/15] feat: move approval dialg into action hitory

---
 src/common/App.tsx            |  2 -
 src/common/ApprovalDialog.tsx | 34 -----------------
 src/common/TaskHistory.tsx    | 72 ++++++++++++++++++++++++++++++++---
 3 files changed, 66 insertions(+), 42 deletions(-)
 delete mode 100644 src/common/ApprovalDialog.tsx

diff --git a/src/common/App.tsx b/src/common/App.tsx
index 555dc50..24d2e33 100644
--- a/src/common/App.tsx
+++ b/src/common/App.tsx
@@ -14,7 +14,6 @@ import { useAppState } from "../state/store";
 import SetAPIKey from "./settings/SetAPIKey";
 import TaskUI from "./TaskUI";
 import Settings from "./Settings";
-import ApprovalDialog from "./ApprovalDialog";
 
 const App = () => {
   const hasAPIKey = useAppState(
@@ -24,7 +23,6 @@ const App = () => {
 
   return (
     <ChakraProvider>
-      <ApprovalDialog />
       <Box p="8" pb="24" fontSize="lg" w="full">
         <HStack mb={4} alignItems="center">
           <Heading as="h1" size="lg" flex={1}>
diff --git a/src/common/ApprovalDialog.tsx b/src/common/ApprovalDialog.tsx
deleted file mode 100644
index 25fbf4a..0000000
--- a/src/common/ApprovalDialog.tsx
+++ /dev/null
@@ -1,34 +0,0 @@
-import { useAppState } from "../state/store";
-
-const ApprovalDialog = () => {
-  const { isPendingApproval, setUserDecision } = useAppState(
-    (state) => state.hitl,
-  );
-
-  if (!isPendingApproval) return null;
-
-  return (
-    <div className="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
-      <div className="bg-white p-6 rounded-lg shadow-lg max-w-md">
-        <h2 className="text-lg font-bold mb-4">Approval Required</h2>
-        <p className="mb-4">Pending Approval Box Placeholder</p>
-        <div className="flex justify-end gap-2">
-          <button
-            className="px-4 py-2 bg-red-500 text-white rounded"
-            onClick={() => setUserDecision("reject")}
-          >
-            Reject
-          </button>
-          <button
-            className="px-4 py-2 bg-green-500 text-white rounded"
-            onClick={() => setUserDecision("approve")}
-          >
-            Approve
-          </button>
-        </div>
-      </div>
-    </div>
-  );
-};
-
-export default ApprovalDialog;
diff --git a/src/common/TaskHistory.tsx b/src/common/TaskHistory.tsx
index 1894143..25bebaf 100644
--- a/src/common/TaskHistory.tsx
+++ b/src/common/TaskHistory.tsx
@@ -16,6 +16,8 @@ import {
   Spacer,
   ColorProps,
   BackgroundProps,
+  Text,
+  Button,
 } from "@chakra-ui/react";
 import { TaskHistoryEntry } from "../state/currentTask";
 import { BsSortNumericDown, BsSortNumericUp } from "react-icons/bs";
@@ -153,6 +155,49 @@ const TaskHistoryItem = ({ index, entry }: TaskHistoryItemProps) => {
   );
 };
 
+const PendingApprovalItem = () => {
+  const { isPendingApproval, proposedAction, setUserDecision } = useAppState(
+    (state) => state.hitl,
+  );
+
+  if (!isPendingApproval || !proposedAction) return null;
+
+  return (
+    <Box
+      border="2px solid"
+      borderColor="yellow.400"
+      p="4"
+      backgroundColor="yellow.50"
+    >
+      <VStack align="stretch" spacing={4}>
+        <HStack>
+          <Box mr="4" fontWeight="bold">
+            ⚠️
+          </Box>
+          <Text fontWeight="medium">Action requires approval</Text>
+        </HStack>
+        <Text fontSize="sm">{proposedAction.thought}</Text>
+        <HStack justify="end" spacing={2}>
+          <Button
+            size="sm"
+            colorScheme="red"
+            onClick={() => setUserDecision("reject")}
+          >
+            Reject
+          </Button>
+          <Button
+            size="sm"
+            colorScheme="green"
+            onClick={() => setUserDecision("approve")}
+          >
+            Approve
+          </Button>
+        </HStack>
+      </VStack>
+    </Box>
+  );
+};
+
 export default function TaskHistory() {
   const { taskHistory, taskStatus } = useAppState((state) => ({
     taskStatus: state.currentTask.status,
@@ -164,16 +209,27 @@ export default function TaskHistory() {
   };
 
   if (taskHistory.length === 0 && taskStatus !== "running") return null;
-  const historyItems = taskHistory.map((entry, index) => (
+
+  // Sort task history based on the sort state
+  const sortedHistory = [...taskHistory].sort((a, b) => {
+    // Assuming each entry has a unique identifier or timestamp for sorting
+    // Replace 'timestamp' with the appropriate property if different
+    const aTime = a.usage?.prompt_tokens || 0;
+    const bTime = b.usage?.prompt_tokens || 0;
+
+    if (sortNumericDown) {
+      return aTime - bTime; // Ascending
+    } else {
+      return bTime - aTime; // Descending
+    }
+  });
+
+  const historyItems = sortedHistory.map((entry, index) => (
     <TaskHistoryItem key={index} index={index} entry={entry} />
   ));
-  historyItems.unshift(<MatchedNotes key="matched-notes" />);
-  if (!sortNumericDown) {
-    historyItems.reverse();
-  }
 
   return (
-    <VStack mt={8}>
+    <VStack mt={8} align="stretch">
       <HStack w="full">
         <Heading as="h3" size="md">
           Action History
@@ -189,6 +245,10 @@ export default function TaskHistory() {
         <CopyButton text={JSON.stringify(taskHistory, null, 2)} />
       </HStack>
       <Accordion allowMultiple w="full" pb="4">
+        {/* Always render fixed items first */}
+        <PendingApprovalItem />
+        <MatchedNotes />
+        {/* Then render sorted history items */}
         {historyItems}
       </Accordion>
     </VStack>

From 200311ba9de450d659cb9aab9df3f039e58b8a00 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 21:42:54 +0100
Subject: [PATCH 12/15] feat: remove 'pattern' field from HITL rules

---
 src/common/HITLSettings/CheckpointItem.tsx |  3 ---
 src/common/HITLSettings/NewHITLForm.tsx    | 18 +++---------------
 src/helpers/hitl/index.ts                  |  1 -
 3 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/src/common/HITLSettings/CheckpointItem.tsx b/src/common/HITLSettings/CheckpointItem.tsx
index 21ea04f..b641f96 100644
--- a/src/common/HITLSettings/CheckpointItem.tsx
+++ b/src/common/HITLSettings/CheckpointItem.tsx
@@ -14,9 +14,6 @@ const CheckpointItem = ({ rule, onEdit, onDelete }: CheckpointItemProps) => {
       <Flex alignItems="flex-start">
         <Box flex="1">
           <Text fontWeight="bold">{rule.description}</Text>
-          <Text fontSize="sm" color="gray.600">
-            Pattern: {rule.pattern}
-          </Text>
         </Box>
         <Box>
           <Tooltip label="Edit checkpoint">
diff --git a/src/common/HITLSettings/NewHITLForm.tsx b/src/common/HITLSettings/NewHITLForm.tsx
index 13ef06e..06a7b15 100644
--- a/src/common/HITLSettings/NewHITLForm.tsx
+++ b/src/common/HITLSettings/NewHITLForm.tsx
@@ -9,7 +9,6 @@ import {
   FormControl,
   FormLabel,
   Input,
-  Textarea,
   Button,
   VStack,
 } from "@chakra-ui/react";
@@ -19,11 +18,10 @@ type NewHITLFormProps = {
   isEditMode: boolean;
   editRule?: {
     id: string;
-    pattern: string;
     description: string;
   };
   closeForm: () => void;
-  onSave: (rule: { pattern: string; description: string }) => void;
+  onSave: (rule: { description: string }) => void;
 };
 
 const NewHITLForm: React.FC<NewHITLFormProps> = ({
@@ -33,13 +31,11 @@ const NewHITLForm: React.FC<NewHITLFormProps> = ({
   closeForm,
   onSave,
 }) => {
-  const [pattern, setPattern] = useState(editRule?.pattern || "");
   const [description, setDescription] = useState(editRule?.description || "");
 
   const handleSubmit = () => {
-    if (!pattern || !description) return;
-    onSave({ pattern, description });
-    setPattern("");
+    if (!description) return;
+    onSave({ description });
     setDescription("");
   };
 
@@ -61,14 +57,6 @@ const NewHITLForm: React.FC<NewHITLFormProps> = ({
                 placeholder="e.g., Confirm before posting to Facebook"
               />
             </FormControl>
-            <FormControl isRequired>
-              <FormLabel>Pattern</FormLabel>
-              <Textarea
-                value={pattern}
-                onChange={(e) => setPattern(e.target.value)}
-                placeholder="e.g., Confirm before clicking 'Post' button on facebook.com"
-              />
-            </FormControl>
             <Button colorScheme="blue" w="full" onClick={handleSubmit}>
               {isEditMode ? "Update" : "Add"} Checkpoint
             </Button>
diff --git a/src/helpers/hitl/index.ts b/src/helpers/hitl/index.ts
index 026730a..9490459 100644
--- a/src/helpers/hitl/index.ts
+++ b/src/helpers/hitl/index.ts
@@ -18,7 +18,6 @@ You response will be strictly "true" or "false". It will not contain any further
 
 export type CheckpointRule = {
   id: string;
-  pattern: string;
   description: string;
 };
 

From c3a7788ebbf338afc265911f27915487368c7ecd Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 21:52:14 +0100
Subject: [PATCH 13/15] fix: shorten placeholder input for checkpoint

---
 src/common/HITLSettings/NewHITLForm.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/HITLSettings/NewHITLForm.tsx b/src/common/HITLSettings/NewHITLForm.tsx
index 06a7b15..ac540ed 100644
--- a/src/common/HITLSettings/NewHITLForm.tsx
+++ b/src/common/HITLSettings/NewHITLForm.tsx
@@ -54,7 +54,7 @@ const NewHITLForm: React.FC<NewHITLFormProps> = ({
               <Input
                 value={description}
                 onChange={(e) => setDescription(e.target.value)}
-                placeholder="e.g., Confirm before posting to Facebook"
+                placeholder="e.g., Confirm before posting a Tweet"
               />
             </FormControl>
             <Button colorScheme="blue" w="full" onClick={handleSubmit}>

From 5fc55f9126d4ed7c6b888a4812066d90bf59fee1 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 22:12:31 +0100
Subject: [PATCH 14/15] fix: approval dialog breaking numberin of action
 history

---
 src/common/TaskHistory.tsx | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/src/common/TaskHistory.tsx b/src/common/TaskHistory.tsx
index 25bebaf..e989173 100644
--- a/src/common/TaskHistory.tsx
+++ b/src/common/TaskHistory.tsx
@@ -77,7 +77,8 @@ const CollapsibleComponent = (props: {
       <AccordionButton>
         <HStack flex="1">
           <Box>{props.title}</Box>
-          <CopyButton text={props.text} /> <Spacer />
+          <CopyButton text={props.text} />
+          <Spacer />
           {props.subtitle && (
             <Box as="span" fontSize="xs" color="gray.500" mr={4}>
               {props.subtitle}
@@ -168,6 +169,7 @@ const PendingApprovalItem = () => {
       borderColor="yellow.400"
       p="4"
       backgroundColor="yellow.50"
+      mb={4}
     >
       <VStack align="stretch" spacing={4}>
         <HStack>
@@ -210,26 +212,24 @@ export default function TaskHistory() {
 
   if (taskHistory.length === 0 && taskStatus !== "running") return null;
 
-  // Sort task history based on the sort state
-  const sortedHistory = [...taskHistory].sort((a, b) => {
-    // Assuming each entry has a unique identifier or timestamp for sorting
-    // Replace 'timestamp' with the appropriate property if different
-    const aTime = a.usage?.prompt_tokens || 0;
-    const bTime = b.usage?.prompt_tokens || 0;
-
-    if (sortNumericDown) {
-      return aTime - bTime; // Ascending
-    } else {
-      return bTime - aTime; // Descending
-    }
-  });
-
-  const historyItems = sortedHistory.map((entry, index) => (
+  // Build the basic history items
+  const historyItems = taskHistory.map((entry, index) => (
     <TaskHistoryItem key={index} index={index} entry={entry} />
   ));
 
+  // Insert matched notes at the top
+  historyItems.unshift(<MatchedNotes key="matched-notes" />);
+
+  // Reverse if needed
+  if (!sortNumericDown) {
+    historyItems.reverse();
+  }
+
   return (
     <VStack mt={8} align="stretch">
+      {/* Pending approval item goes above the heading */}
+      <PendingApprovalItem />
+
       <HStack w="full">
         <Heading as="h3" size="md">
           Action History
@@ -245,10 +245,6 @@ export default function TaskHistory() {
         <CopyButton text={JSON.stringify(taskHistory, null, 2)} />
       </HStack>
       <Accordion allowMultiple w="full" pb="4">
-        {/* Always render fixed items first */}
-        <PendingApprovalItem />
-        <MatchedNotes />
-        {/* Then render sorted history items */}
         {historyItems}
       </Accordion>
     </VStack>

From 05b99633ef5e993278542c54c20480fb4b39d751 Mon Sep 17 00:00:00 2001
From: Emil Schmitz <e.schmitz@outlook.com>
Date: Mon, 27 Jan 2025 23:54:51 +0100
Subject: [PATCH 15/15] feat: add previous actions to hitl system input

---
 src/helpers/hitl/index.ts | 11 +++++++++--
 src/state/currentTask.ts  |  5 ++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/helpers/hitl/index.ts b/src/helpers/hitl/index.ts
index 9490459..2bb30d3 100644
--- a/src/helpers/hitl/index.ts
+++ b/src/helpers/hitl/index.ts
@@ -1,6 +1,7 @@
 import { useAppState } from "../../state/store";
 import { fetchResponseFromModel } from "../aiSdkUtils";
 import { QueryResult } from "../vision-agent/determineNextAction";
+import { Action } from "../vision-agent/parseResponse";
 
 const systemMessage = `
 You are an oversight system for a browser automation assistant.
@@ -8,10 +9,14 @@ You are an oversight system for a browser automation assistant.
 You will receive a JSON with two fields:
   hitl_rule: a rule that describes what kind of actions require human approval before execution. These actions trigger a safety checkpoint.
   action: a JSON describing an action that the assistant intends to execute.
+  previous_actions: an array of previous actions that have been performed, so that you understand what's happening in the current one.
 
 You will be asked to determine if the assistant's intended action falls within the scope of actions described by the rule.
-If they do, then they require human approval and you should respond with "true".
-If they do not, then they do not require human approval and you should respond with "false".
+For the action to trigger the rule, the action must fall precisely within the rule's scope.
+If the action is just leading up to an action requiring approval, it does not require approval itself.
+
+If the action falls within the scope of the rule, then they require human approval and you should respond with "true".
+If it does not, then they do not require human approval and you should respond with "false".
 
 You response will be strictly "true" or "false". It will not contain any further text, including any additional context or explanation.
 `;
@@ -29,6 +34,7 @@ export function hasCheckpointRules(): boolean {
 // TODO: remove console logs
 export async function matchesCheckpointRule(
   query: QueryResult,
+  previousActions: Action[],
 ): Promise<boolean> {
   // If no rules defined, don't require approval
   if (!hasCheckpointRules()) {
@@ -46,6 +52,7 @@ export async function matchesCheckpointRule(
       {
         hitl_rule: rule.description,
         action: rawResponse,
+        previous_actions: previousActions.map((action) => action.thought),
       },
       null,
       2,
diff --git a/src/state/currentTask.ts b/src/state/currentTask.ts
index 85e3f09..3ddc099 100644
--- a/src/state/currentTask.ts
+++ b/src/state/currentTask.ts
@@ -269,7 +269,10 @@ export const createCurrentTaskSlice: MyStateCreator<CurrentTaskSlice> = (
 
           if (wasStopped()) break;
 
-          const needsApproval = await matchesCheckpointRule(query);
+          const needsApproval = await matchesCheckpointRule(
+            query,
+            previousActions,
+          );
 
           if (needsApproval && query) {
             set((state) => {