Merge pull request #3 from Constantinople-AI/GGGG-370-workers-model-cache

Variable workers & model cache

Author: ofawx

src/Dockerfile_ecs

@@ -9,5 +9,6 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
 COPY ./api /app/api
 
 ENV PORT=80
+ENV WORKERS=1
 
-CMD ["sh", "-c", "uvicorn api.app:app --host 0.0.0.0 --port ${PORT}"]
+CMD ["sh", "-c", "uvicorn api.app:app --host 0.0.0.0 --port ${PORT} --workers ${WORKERS}"]

src/api/models/bedrock.py

@@ -46,6 +46,7 @@
     ENABLE_CROSS_REGION_INFERENCE,
     ENABLE_APPLICATION_INFERENCE_PROFILES,
     MAX_RETRIES_AWS,
+    MODEL_CACHE_TTL,
 )
 
 logger = logging.getLogger(__name__)
@@ -171,15 +172,40 @@ def list_bedrock_models() -> dict:
     return model_list
 
 
+# In-memory cache
+_model_cache = {
+    "data": None,
+    "timestamp": 0
+}
+
+def _get_cached_models():
+    """Get models from in-memory cache if still valid."""
+    global _model_cache
+    
+    current_time = time.time()
+    cache_age = current_time - _model_cache["timestamp"]
+    
+    if _model_cache["data"] is None or cache_age > MODEL_CACHE_TTL:
+        fresh_models = list_bedrock_models()
+        if fresh_models:
+            _model_cache["data"] = fresh_models
+            _model_cache["timestamp"] = current_time
+        return fresh_models
+    else:
+        # Cache hit
+        return _model_cache["data"]
+
 # Initialize the model list.
-bedrock_model_list = list_bedrock_models()
+bedrock_model_list = _get_cached_models()
 
 
 class BedrockModel(BaseChatModel):
     def list_models(self) -> list[str]:
-        """Always refresh the latest model list"""
+        """Get model list using in-memory cache with TTL"""
         global bedrock_model_list
-        bedrock_model_list = list_bedrock_models()
+        cached_models = _get_cached_models()
+        if cached_models:
+            bedrock_model_list = cached_models
         return list(bedrock_model_list.keys())
 
     def validate(self, chat_request: ChatRequest):

src/api/setting.py

@@ -20,3 +20,4 @@
 ENABLE_CROSS_REGION_INFERENCE = os.environ.get("ENABLE_CROSS_REGION_INFERENCE", "true").lower() != "false"
 MAX_RETRIES_AWS = int(os.environ.get("MAX_RETRIES_AWS", "3"))
 ENABLE_APPLICATION_INFERENCE_PROFILES = os.environ.get("ENABLE_APPLICATION_INFERENCE_PROFILES", "true").lower() != "false"
+MODEL_CACHE_TTL = int(os.environ.get("MODEL_CACHE_TTL", "3600"))  # 1 hour default