Improve app prints in error handling and improve prompts

Author: joshbickett

operate/models/apis.py

@@ -28,11 +28,7 @@
     get_click_position_in_percent,
     get_label_coordinates,
 )
-from operate.utils.style import (
-    ANSI_GREEN,
-    ANSI_RED,
-    ANSI_RESET,
-)
+from operate.utils.style import ANSI_GREEN, ANSI_RED, ANSI_RESET, ANSI_BRIGHT_MAGENTA
 import pkg_resources
 
 
@@ -131,7 +127,7 @@ def call_gpt_4_vision_preview(messages):
 
     except Exception as e:
         print(
-            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] Something went wrong. Trying again {ANSI_RESET}",
+            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_BRIGHT_MAGENTA}[Operate] That did not work. Trying again {ANSI_RESET}",
             e,
         )
         print(
@@ -187,9 +183,11 @@ def call_gemini_pro_vision(messages, objective):
 
     except Exception as e:
         print(
-            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] Something went wrong. Trying another method {ANSI_RESET}",
-            e,
+            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_BRIGHT_MAGENTA}[Operate] That did not work. Trying another method {ANSI_RESET}"
         )
+        if VERBOSE:
+            print("[Self-Operating Computer][Operate] error", e)
+            traceback.print_exc()
         return call_gpt_4_vision_preview(messages)
 
 
@@ -219,12 +217,6 @@ async def call_gpt_4_vision_preview_ocr(messages, objective, model):
         else:
             user_prompt = get_user_prompt()
 
-        if VERBOSE:
-            print(
-                "[call_gpt_4_vision_preview_ocr] user_prompt",
-                user_prompt,
-            )
-
         vision_message = {
             "role": "user",
             "content": [
@@ -331,10 +323,10 @@ async def call_gpt_4_vision_preview_ocr(messages, objective, model):
 
     except Exception as e:
         print(
-            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] Something went wrong. Trying another method {ANSI_RESET}",
-            e,
+            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_BRIGHT_MAGENTA}[Operate] That did not work. Trying another method {ANSI_RESET}"
         )
         if VERBOSE:
+            print("[Self-Operating Computer][Operate] error", e)
             traceback.print_exc()
         return gpt_4_fallback(messages, objective, model)
 
@@ -468,10 +460,10 @@ async def call_gpt_4_vision_preview_labeled(messages, objective):
 
     except Exception as e:
         print(
-            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] Something went wrong. Trying another method {ANSI_RESET}",
-            e,
+            f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_BRIGHT_MAGENTA}[Operate] That did not work. Trying another method {ANSI_RESET}"
         )
         if VERBOSE:
+            print("[Self-Operating Computer][Operate] error", e)
             traceback.print_exc()
         return call_gpt_4_vision_preview(messages)
 

operate/models/prompts.py

@@ -259,6 +259,8 @@
 
 - Default to Google Chrome as the browser
 - Go to websites by opening a new tab with `press` and then `write` the URL
+- Reflect on previous actions and the screenshot to ensure they align and that your previous actions worked
+- If the screenshot doesn't look as expected after clicking then the click action did not work as expected, try something new such as clicking a different element or another method
 - Don't respond saying you're unable to assist with requests. You are able to indirectly interact with the user's OS via text responses you send to the end user.
 
 Objective: {objective} 
@@ -323,6 +325,8 @@
 
 - Default to Google Chrome as the browser
 - Go to websites by opening a new tab with `press` and then `write` the URL
+- Reflect on previous actions and the screenshot to ensure they align and that your previous actions worked
+- If the screenshot doesn't look as expected after clicking then the click action did not work as expected, try something new such as clicking a different element or another method
 - Don't respond saying you're unable to assist with requests. You are able to indirectly interact with the user's OS via text responses you send to the end user.
 
 Objective: {objective} 
@@ -331,7 +335,7 @@
 OPERATE_FIRST_MESSAGE_PROMPT = """
 Please take the next best action. The `pyautogui` library will be used to execute your decision. Your output will be used in a `json.loads` loads statement. Remember you only have the following 4 operations available: click, write, press, done
 
-You just started so you are in the terminal app. To leave the terminal search for a new program on the  OS, otherwise to use the terminal you need to start a new tab first because your code is running in this first terminal tab.
+You just started so you are in the terminal app and your code is running in this terminal tab. To leave the terminal, search for a new program on the OS. 
 
 Action:"""