Refactor Processing Pipeline

.gitignore

@@ -2,4 +2,5 @@ __pycache__/
 **/Images/
 **.DS_Store
 **/flagged/
-.env
+.env
+settings.json

HybridSearch_example/docker-compose.yml

@@ -75,6 +75,23 @@ services:
               count: all
               capabilities: [gpu]
 
+  weavloader:
+    build:
+      context: ./weavloader
+    ports:
+      - 8080:8080  # Metrics port
+    restart: on-failure
+    environment:
+      SAGE_USER: ${SAGE_USER}
+      SAGE_PASS: ${SAGE_PASS}
+      TRITON_HOST: 'triton'
+      TRITON_PORT: '8001'
+      WEAVIATE_HOST: 'weaviate'
+      WEAVIATE_PORT: '8080'
+    depends_on:
+      - weaviate
+      - triton
+
   gradio-ui:
     build:
       context: ./app

HybridSearch_example/kubernetes/README.md

@@ -0,0 +1 @@
+This folder contains the nautilous Kubernetes deployment manifests for the Sage Hybrid Search system. 

HybridSearch_example/weavloader/Dockerfile

@@ -5,7 +5,9 @@ RUN apt-get update \
   wget \
   vim \
   jq \
-  curl
+  curl \
+  redis-server \
+  supervisor
 
 # Set working directory
 WORKDIR /app
@@ -18,9 +20,26 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 # Set environment variables
 ENV MANIFEST_API=https://auth.sagecontinuum.org/manifests/
+ENV CELERY_BROKER_URL=redis://localhost:6379/0
+ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0
+ENV PYTHONUNBUFFERED=1
+ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc_dir
 
 # Copy the rest of the application code into the container
 COPY . .
 
-# Run upload.py
-CMD ["python", "main.py"]
+# Create supervisor configuration
+RUN mkdir -p /var/log/supervisor
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# Create Redis data directory with proper permissions
+RUN mkdir -p /var/lib/redis && chown -R root:root /var/lib/redis && chmod 755 /var/lib/redis
+
+# Create Prometheus multiprocess directory
+RUN mkdir -p /tmp/prometheus_multiproc_dir && chmod 755 /tmp/prometheus_multiproc_dir
+
+# Expose ports
+EXPOSE 8080
+
+# Use supervisor to manage both Redis and Celery
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]