Remove unnecessary env var & reposition env var source (#1184)

(cherry picked from commit 8843326)

Author: RobertLucian

cli/local/docker_spec.go

@@ -93,7 +93,7 @@ func getAPIEnv(api *spec.API, awsClient *aws.Client) []string {
 		"CORTEX_THREADS_PER_PROCESS="+s.Int32(api.Predictor.ThreadsPerProcess),
 		// add 1 because it was required to achieve the target concurrency for 1 process, 1 thread
 		"CORTEX_MAX_PROCESS_CONCURRENCY="+s.Int64(1+int64(math.Round(float64(consts.DefaultMaxReplicaConcurrency)/float64(api.Predictor.ProcessesPerReplica)))),
-		"CORTEX_SO_MAX_CONN=1000",
+		"CORTEX_SO_MAX_CONN="+s.Int64(consts.DefaultMaxReplicaConcurrency+100), // add a buffer to be safe
 		"AWS_REGION="+awsClient.Region,
 	)
 

pkg/operator/operator/k8s_specs.go

@@ -624,10 +624,6 @@ func getEnvVars(api *spec.API, container string) []kcore.EnvVar {
 				Name:  "CORTEX_THREADS_PER_PROCESS",
 				Value: s.Int32(api.Predictor.ThreadsPerProcess),
 			},
-			kcore.EnvVar{
-				Name:  "CORTEX_MAX_REPLICA_CONCURRENCY",
-				Value: s.Int64(api.Autoscaling.MaxReplicaConcurrency),
-			},
 			kcore.EnvVar{
 				Name: "CORTEX_MAX_PROCESS_CONCURRENCY",
 				// add 1 because it was required to achieve the target concurrency for 1 process, 1 thread

pkg/workloads/cortex/serve/run.sh

@@ -39,6 +39,13 @@ rm -rf /mnt/workspace/api_readiness.txt
 # allow for the liveness check to pass until the API is running
 echo "9999999999" > /mnt/workspace/api_liveness.txt
 
+# export environment variables
+if [ -f "/mnt/project/.env" ]; then
+    set -a
+    source /mnt/project/.env
+    set +a
+fi
+
 export PYTHONPATH=$PYTHONPATH:$PYTHON_PATH
 # ensure predictor print() statements are always flushed
 export PYTHONUNBUFFERED=TRUE
@@ -49,13 +56,6 @@ if [ "$CORTEX_PROVIDER" != "local" ]; then
     sysctl -w net.ipv4.tcp_fin_timeout=30 >/dev/null
 fi
 
-# export environment variables
-if [ -f "/mnt/project/.env" ]; then
-    set -a
-    source /mnt/project/.env
-    set +a
-fi
-
 # execute script if present in project's directory
 if [ -f "/mnt/project/dependencies.sh" ]; then
     bash -e /mnt/project/dependencies.sh

pkg/workloads/cortex/serve/start_uvicorn.py

@@ -87,7 +87,9 @@ def main():
         host="0.0.0.0",
         port=int(os.environ["CORTEX_SERVING_PORT"]),
         workers=int(os.environ["CORTEX_PROCESSES_PER_REPLICA"]),
-        limit_concurrency=int(os.environ["CORTEX_MAX_PROCESS_CONCURRENCY"]),
+        limit_concurrency=int(
+            os.environ["CORTEX_MAX_PROCESS_CONCURRENCY"]
+        ),  # this is a per process limit
         backlog=int(os.environ["CORTEX_SO_MAX_CONN"]),
         log_config=log_config,
         log_level="info",