add other ollama visual llms

tosquash

to squash w/ first

Author: ketsapiwiq

operate/models/apis.py

@@ -52,8 +52,8 @@ async def get_next_action(model, messages, objective, session_id):
         return "coming soon"
     if model == "gemini-pro-vision":
         return call_gemini_pro_vision(messages, objective), None
-    if model == "llava":
-        operation = call_ollama_llava(messages)
+    if model == "llava" or model == "llava:13b" or "bakllava" or "llava-llama3":
+        operation = call_ollama_llava(messages, model)
         return operation, None
     if model == "claude-3":
         operation = await call_claude_3_with_ocr(messages, objective, model)
@@ -453,9 +453,11 @@ async def call_gpt_4_vision_preview_labeled(messages, objective, model):
         return call_gpt_4_vision_preview(messages)
 
 
-def call_ollama_llava(messages):
+def call_ollama_llava(messages, model):
+    if model == "":
+        model = "llava"
     if config.verbose:
-        print("[call_ollama_llava]")
+        print(f"[call_ollama_llava] model {model}")
     time.sleep(1)
     try:
         screenshots_dir = "screenshots"
@@ -485,7 +487,7 @@ def call_ollama_llava(messages):
         messages.append(vision_message)
 
         response = ollama.chat(
-            model="llava",
+            model=model,
             messages=messages,
         )
 
@@ -527,7 +529,7 @@ def call_ollama_llava(messages):
         )
         if config.verbose:
             traceback.print_exc()
-        return call_ollama_llava(messages)
+        return call_ollama_llava(messages, model)
 
 
 async def call_claude_3_with_ocr(messages, objective, model):