From 9e0ac5b7fef0b4bce3df12bdc0317f461ff72285 Mon Sep 17 00:00:00 2001 From: smore Date: Thu, 19 Sep 2024 14:32:51 -0700 Subject: [PATCH 1/4] Create localai-main-with-logs-sidecar.yaml Add SideCar to grep on logs and get n_tokens_second --- .../k8s/localai-main-with-logs-sidecar.yaml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 configs/k8s/localai-main-with-logs-sidecar.yaml diff --git a/configs/k8s/localai-main-with-logs-sidecar.yaml b/configs/k8s/localai-main-with-logs-sidecar.yaml new file mode 100644 index 0000000..126864b --- /dev/null +++ b/configs/k8s/localai-main-with-logs-sidecar.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: localai-deployment + namespace: localai + labels: + app: localai +spec: + replicas: 1 # Single instance of the main LocalAI + selector: + matchLabels: + app: localai + template: + metadata: + labels: + app: localai + spec: + containers: + - name: localai + image: localai/localai:latest-aio-gpu-nvidia-cuda-12 + ports: + - containerPort: 8080 # HTTP port for API requests + - containerPort: 9000 # P2P port for worker communication + command: + - /bin/sh + - -c + - | + local-ai run --debug --p2p --address 0.0.0.0:8080 > /var/log/localai/localai.log 2>&1 + volumeMounts: + - name: log-volume + mountPath: /var/log/localai + env: + - name: ENABLE_SIDECAR + value: "true" + # Sidecar container that is conditionally active + - name: logs-sidecar + image: busybox + ports: + - containerPort: 8081 # Metrics port for the sidecar + volumeMounts: + - name: log-volume + mountPath: /var/log/localai + # Grep and Tail the most recent occurence of n_tokens_second + # Response back with the latest value on port 8081 + command: + - /bin/sh + - -c + - | + if [ "$(echo $ENABLE_SIDECAR)" = "true" ]; then + while true; do + n_tokens_second=$(grep -o '"n_tokens_second":[^,]*' /var/log/localai/localai.log | tail -n1 | awk -F':' '{print $2}') + echo -e "HTTP/1.1 200 OK\n\n$n_tokens_second" | nc -l -p 8081 -q 1; + done; + else + echo "Sidecar not enabled"; + sleep 3600; + fi + + volumes: + - name: log-volume + emptyDir: {} # Shared volume to store logs From 8c0c2de975928c588fad8669d1d5756bdda69aa7 Mon Sep 17 00:00:00 2001 From: smore Date: Thu, 19 Sep 2024 14:36:01 -0700 Subject: [PATCH 2/4] Update localai-main-with-logs-sidecar.yaml --- configs/k8s/localai-main-with-logs-sidecar.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/k8s/localai-main-with-logs-sidecar.yaml b/configs/k8s/localai-main-with-logs-sidecar.yaml index 126864b..41eae14 100644 --- a/configs/k8s/localai-main-with-logs-sidecar.yaml +++ b/configs/k8s/localai-main-with-logs-sidecar.yaml @@ -32,7 +32,7 @@ spec: env: - name: ENABLE_SIDECAR value: "true" - # Sidecar container that is conditionally active + # Sidecar container for tailing log and getting n_tokens_second - name: logs-sidecar image: busybox ports: From 4c3f22adadfcef958b0083c2e3a7373c1ba15603 Mon Sep 17 00:00:00 2001 From: smore Date: Thu, 19 Sep 2024 15:20:20 -0700 Subject: [PATCH 3/4] Update localai-main-with-logs-sidecar.yaml --- configs/k8s/localai-main-with-logs-sidecar.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/k8s/localai-main-with-logs-sidecar.yaml b/configs/k8s/localai-main-with-logs-sidecar.yaml index 41eae14..14302ad 100644 --- a/configs/k8s/localai-main-with-logs-sidecar.yaml +++ b/configs/k8s/localai-main-with-logs-sidecar.yaml @@ -32,7 +32,7 @@ spec: env: - name: ENABLE_SIDECAR value: "true" - # Sidecar container for tailing log and getting n_tokens_second + # Sidecar container that is conditionally active - name: logs-sidecar image: busybox ports: @@ -40,6 +40,9 @@ spec: volumeMounts: - name: log-volume mountPath: /var/log/localai + env: + - name: ENABLE_SIDECAR + value: "true" # Explicitly set the same environment variable in the sidecar # Grep and Tail the most recent occurence of n_tokens_second # Response back with the latest value on port 8081 command: @@ -49,13 +52,12 @@ spec: if [ "$(echo $ENABLE_SIDECAR)" = "true" ]; then while true; do n_tokens_second=$(grep -o '"n_tokens_second":[^,]*' /var/log/localai/localai.log | tail -n1 | awk -F':' '{print $2}') - echo -e "HTTP/1.1 200 OK\n\n$n_tokens_second" | nc -l -p 8081 -q 1; + echo -e "HTTP/1.1 200 OK\n\n$n_tokens_second" | nc -l -p 8081; done; else echo "Sidecar not enabled"; sleep 3600; fi - volumes: - name: log-volume emptyDir: {} # Shared volume to store logs From 0e0312a40952379963f89e3f006daa00926dc7ba Mon Sep 17 00:00:00 2001 From: smore Date: Thu, 19 Sep 2024 15:34:18 -0700 Subject: [PATCH 4/4] Update localai-main-with-logs-sidecar.yaml --- configs/k8s/localai-main-with-logs-sidecar.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/k8s/localai-main-with-logs-sidecar.yaml b/configs/k8s/localai-main-with-logs-sidecar.yaml index 14302ad..3f466b3 100644 --- a/configs/k8s/localai-main-with-logs-sidecar.yaml +++ b/configs/k8s/localai-main-with-logs-sidecar.yaml @@ -25,7 +25,7 @@ spec: - /bin/sh - -c - | - local-ai run --debug --p2p --address 0.0.0.0:8080 > /var/log/localai/localai.log 2>&1 + /build/local-ai run --debug --p2p --address 0.0.0.0:8080 > /var/log/localai/localai.log 2>&1 volumeMounts: - name: log-volume mountPath: /var/log/localai