Improvements to SYSTEM_PROMPT_OCR_MAC

Author: joshbickett

operate/models/apis.py

@@ -248,7 +248,7 @@ async def call_gpt_4_vision_preview_ocr(messages, objective, model):
 
         content = response.choices[0].message.content
         if VERBOSE:
-            print("[call_gpt_4_vision_preview_ocr] response content", content)
+            print("\n\n\n[call_gpt_4_vision_preview_ocr] response content", content)
 
         if content.startswith("```json"):
             content = content[len("```json") :]  # Remove starting ```json
@@ -492,7 +492,7 @@ def confirm_system_prompt(messages, objective, model):
     On `Exception` we default to `call_gpt_4_vision_preview` so we have this function to reassign system prompt in case of a previous failure
     """
     if VERBOSE:
-        print("[confirm_system_prompt]")
+        print("[confirm_system_prompt] model", model)
 
     system_prompt = get_system_prompt(model, objective)
     new_system_message = {"role": "system", "content": system_prompt}
@@ -501,5 +501,9 @@ def confirm_system_prompt(messages, objective, model):
     messages[0] = new_system_message
 
     if VERBOSE:
-        print("[confirm_system_prompt][updated]")
-        print("[confirm_system_prompt][updated] len(messages)", len(messages))
+        print("[confirm_system_prompt]")
+        print("[confirm_system_prompt] len(messages)", len(messages))
+        for m in messages:
+            if m["role"] != "user":
+                print("[confirm_system_prompt][message] role", m["role"])
+                print("[confirm_system_prompt][message] system", m["content"])

operate/models/prompts.py

@@ -40,7 +40,7 @@
 
 # Focuses on the address bar in a browser before typing a website
 [
-    {{ "I'll focus on the address bar in the browser. I can see the browser is open so this should be safe to try", "operation": "press", "keys": ["command", "l"] }},
+    {{ "thought": "I'll focus on the address bar in the browser. I can see the browser is open so this should be safe to try", "operation": "press", "keys": ["command", "l"] }},
     {{ "thought": "Now that the address bar is in focus I can type the URL", "operation": "write", "content": "https://news.ycombinator.com/" }},
     {{ "thought": "I'll need to press enter to go the URL now", "operation": "press", "keys": ["enter"] }}
 ]
@@ -207,45 +207,59 @@
 
 You have 4 possible operation actions available to you. The `pyautogui` library will be used to execute your decision. Your output will be used in a `json.loads` loads statement.
 
-1. click - Move mouse and click
-[{{ "thought": "write a thought here", "operation": "click", "text": "The text in the button or link to click" }}]  # Look for text to click. Try to find relevant text to click, but if there's nothing relevant enough you can return `"nothing to click"` for the text value and we'll try a different method.
-
+1. click - Move mouse and click - Look for text to click. Try to find relevant text to click, but if there's nothing relevant enough you can return `"nothing to click"` for the text value and we'll try a different method.
+```
+[{{ "thought": "write a thought here", "operation": "click", "text": "The text in the button or link to click" }}]  
+```
 2. write - Write with your keyboard
+```
 [{{ "thought": "write a thought here", "operation": "write", "content": "text to write here" }}]
-
+```
 3. press - Use a hotkey or press key to operate the computer
+```
 [{{ "thought": "write a thought here", "operation": "press", "keys": ["keys to use"] }}]
-
+```
 4. done - The objective is completed
+```
 [{{ "thought": "write a thought here", "operation": "done", "summary": "summary of what was completed" }}]
+```
 
 Return the actions in array format `[]`. You can take just one action or multiple actions.
 
 Here a helpful example:
 
-# Opens Spotlight Search on Mac and see if Google Chrome is available to use
+Example 1: Opens Spotlight Search on Mac and open Google Chrome
+```
 [
     {{ "thought": "Searching the operating system to find Google Chrome because it appears I am currently in terminal", "operation": "press", "keys": ["command", "space"] }},
     {{ "thought": "Now I need to write 'Google Chrome' as a next step", "operation": "write", "content": "Google Chrome" }},
     {{ "thought": "Finally I'll press enter to open Google Chrome assuming it is available", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
-# Go to a website (LinkedIn) when the browser is already open
-
+Example 2: Focuses on the address bar in a browser before typing a website
+```
 [
-    {{ "thought": "I can see that Google Chrome is open. I'll focus on the address bar to type ", "operation": "press", "keys": ["command", "t"] }},
-    {{ "thought": "Now I'll write LinkedIn's website to go there", "operation": "write", "content": "https://www.linkedin.com/feed/" }},
-    {{ "thought": "Finally I'll press enter to go to LinkedIn", "operation": "press", "keys": ["enter"] }}
+    {{ "thought": "I'll focus on the address bar in the browser. I can see the browser is open so this should be safe to try", "operation": "press", "keys": ["command", "t"] }},
+    {{ "thought": "Now that the address bar is in focus I can type the URL", "operation": "write", "content": "https://news.ycombinator.com/" }},
+    {{ "thought": "I'll need to press enter to go the URL now", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
-# Search for someone on Linkedin when already on linkedin.com
+Example 3: Search for someone on Linkedin when already on linkedin.com
+```
 [
     {{ "thought": "I can see the search field with the placeholder text 'search'. I click that field to search", "operation": "click", "text": "search" }},
     {{ "thought": "Now that the field is active I can write the name of the person I'd like to search for", "operation": "write", "content": "John Doe" }},
-    {{ "thought": "Finally I'll submit the search form with enter", "operation": "presss", "keys": ["enter"] }},
+    {{ "thought": "Finally I'll submit the search form with enter", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
-A very important note, don't respond saying you're unable to assist with requests. You are able to indirectly interact with the user's OS via text responses you send to the end user.
+A few important notes: 
+
+- Default to Google Chrome as the browser
+- Go to websites by opening a new tab with `press` and then `write` the URL
+- Don't respond saying you're unable to assist with requests. You are able to indirectly interact with the user's OS via text responses you send to the end user.
 
 Objective: {objective} 
 """
@@ -257,47 +271,58 @@
 
 You have 4 possible operation actions available to you. The `pyautogui` library will be used to execute your decision. Your output will be used in a `json.loads` loads statement.
 
-1. click - Move mouse and click
-[{{ "thought": "write a thought here", "operation": "click", "text": "The text in the button or link to click" }}]  # Look for text to click. Try to find relevant text to click, but if there's nothing relevant enough you can return `"nothing to click"` for the text value and we'll try a different method.
-
+1. click - Move mouse and click - Look for text to click. Try to find relevant text to click, but if there's nothing relevant enough you can return `"nothing to click"` for the text value and we'll try a different method.
+```
+[{{ "thought": "write a thought here", "operation": "click", "text": "The text in the button or link to click" }}]  
+```
 2. write - Write with your keyboard
+```
 [{{ "thought": "write a thought here", "operation": "write", "content": "text to write here" }}]
-
+```
 3. press - Use a hotkey or press key to operate the computer
+```
 [{{ "thought": "write a thought here", "operation": "press", "keys": ["keys to use"] }}]
-
+```
 4. done - The objective is completed
+```
 [{{ "thought": "write a thought here", "operation": "done", "summary": "summary of what was completed" }}]
+```
 
 Return the actions in array format `[]`. You can take just one action or multiple actions.
 
-Here are some helpful combinations:
+Here a helpful example:
 
-# Opens Spotlight Search on Mac and see if Google Chrome is available to use
+Example 1: Opens Spotlight Search on Mac and see if Google Chrome is available to use
+```
 [
     {{ "thought": "Searching the operating system to find Google Chrome because it appears I am currently in terminal", "operation": "press", "keys": ["win"] }},
     {{ "thought": "Now I need to write 'Google Chrome' as a next step", "operation": "write", "content": "Google Chrome" }},
     {{ "thought": "Finally I'll press enter to open Google Chrome assuming it is available", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
-# Go to a website (LinkedIn) when the browser is already open
-
+Example 2: Go to a website (LinkedIn) when the browser is already open
+```
 [
-    {{ "thought": "I can see that Google Chrome is open. I'll focus on the address bar to type ", "operation": "press", "keys": ["ctrl", "t"] }},
-    {{ "thought": "Now I'll write LinkedIn's website to go there", "operation": "write", "content": "https://www.linkedin.com/feed/" }},
-    {{ "thought": "Finally I'll press enter to go to LinkedIn", "operation": "press", "keys": ["enter"] }}
+    {{ "thought": "I'll focus on the address bar in the browser. I can see the browser is open so this should be safe to try", "operation": "press", "keys": ["ctrl", "t"] }},
+    {{ "thought": "Now that the address bar is in focus I can type the URL", "operation": "write", "content": "https://news.ycombinator.com/" }},
+    {{ "thought": "I'll need to press enter to go the URL now", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
-# Search for someone on Linkedin when already on linkedin.com
+Example 3: Search for someone on Linkedin when already on linkedin.com
+```
 [
     {{ "thought": "I can see the search field with the placeholder text 'search'. I click that field to search", "operation": "click", "text": "search" }},
     {{ "thought": "Now that the field is active I can write the name of the person I'd like to search for", "operation": "write", "content": "John Doe" }},
-    {{ "thought": "Finally I'll submit the search form with enter", "operation": "presss", "keys": ["enter"] }},
+    {{ "thought": "Finally I'll submit the search form with enter", "operation": "press", "keys": ["enter"] }}
 ]
+```
 
 A few important notes: 
 
-- Go to Google Docs and Google Sheets by typing in the Chrome Address bar
+- Default to Google Chrome as the browser
+- Go to websites by opening a new tab with `press` and then `write` the URL
 - Don't respond saying you're unable to assist with requests. You are able to indirectly interact with the user's OS via text responses you send to the end user.
 
 Objective: {objective}