Restart deployments depending on cluster config during cx configure (#2367)

Author: vishalbollu

cmd/activator/main.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/cortexlabs/cortex/pkg/activator"
@@ -32,7 +33,6 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/k8s"
 	"github.com/cortexlabs/cortex/pkg/lib/logging"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
-	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
 	"go.uber.org/zap"
 	istioinformers "istio.io/client-go/pkg/informers/externalversions"
@@ -43,12 +43,11 @@ import (
 
 func main() {
 	var (
-		port              int
-		adminPort         int
-		inCluster         bool
-		autoscalerURL     string
-		namespace         string
-		clusterConfigPath string
+		port          int
+		adminPort     int
+		inCluster     bool
+		autoscalerURL string
+		namespace     string
 	)
 
 	flag.IntVar(&port, "port", 8000, "port where the activator server will be exposed")
@@ -59,7 +58,6 @@ func main() {
 		"kubernetes namespace where the cortex APIs are deployed "+
 			"(can be set through the CORTEX_NAMESPACE env variable)",
 	)
-	flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path")
 	flag.Parse()
 
 	log := logging.GetLogger()
@@ -72,16 +70,9 @@ func main() {
 		log.Fatal("--autoscaler-url is a required option")
 	case namespace == "":
 		log.Fatal("--namespace is a required option")
-	case clusterConfigPath == "":
-		log.Fatal("--cluster-config flag is required")
 	}
 
-	clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)
-	if err != nil {
-		exit(log, err)
-	}
-
-	awsClient, err := aws.NewForRegion(clusterConfig.Region)
+	awsClient, err := aws.New()
 	if err != nil {
 		exit(log, err)
 	}
@@ -91,8 +82,10 @@ func main() {
 		exit(log, err)
 	}
 
+	telemetryEnabled := strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) != "true"
+
 	err = telemetry.Init(telemetry.Config{
-		Enabled: clusterConfig.Telemetry,
+		Enabled: telemetryEnabled,
 		UserID:  userID,
 		Properties: map[string]string{
 			"kind":       userconfig.RealtimeAPIKind.String(),

cmd/autoscaler/main.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/cortexlabs/cortex/pkg/autoscaler"
@@ -32,7 +33,6 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/k8s"
 	"github.com/cortexlabs/cortex/pkg/lib/logging"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
-	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
 	"github.com/gorilla/mux"
 	promapi "github.com/prometheus/client_golang/api"
@@ -49,11 +49,10 @@ import (
 
 func main() {
 	var (
-		port              int
-		inCluster         bool
-		prometheusURL     string
-		namespace         string
-		clusterConfigPath string
+		port          int
+		inCluster     bool
+		prometheusURL string
+		namespace     string
 	)
 
 	flag.IntVar(&port, "port", 8000, "port where the autoscaler server will be exposed")
@@ -65,7 +64,6 @@ func main() {
 		"kubernetes namespace where the cortex APIs are deployed "+
 			"(can be set through the CORTEX_NAMESPACE env variable)",
 	)
-	flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path")
 	flag.Parse()
 
 	log := logging.GetLogger()
@@ -78,16 +76,9 @@ func main() {
 		log.Fatal("--prometheus-url is a required option")
 	case namespace == "":
 		log.Fatal("--namespace is a required option")
-	case clusterConfigPath == "":
-		log.Fatal("--cluster-config flag is required")
 	}
 
-	clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)
-	if err != nil {
-		exit(log, err)
-	}
-
-	awsClient, err := aws.NewForRegion(clusterConfig.Region)
+	awsClient, err := aws.New()
 	if err != nil {
 		exit(log, err)
 	}
@@ -97,8 +88,10 @@ func main() {
 		exit(log, err)
 	}
 
+	telemetryEnabled := strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) != "true"
+
 	err = telemetry.Init(telemetry.Config{
-		Enabled: clusterConfig.Telemetry,
+		Enabled: telemetryEnabled,
 		UserID:  userID,
 		Properties: map[string]string{
 			"kind":       userconfig.RealtimeAPIKind.String(),

manager/install.sh

@@ -103,6 +103,8 @@ function cluster_configure() {
   python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/cluster-autoscaler.yaml.j2 | kubectl apply -f - >/dev/null
   echo "✓"
 
+  restart_controller_manager
+
   restart_operator
 
   validate_cortex
@@ -276,6 +278,14 @@ function start_controller_manager() {
   echo "✓"
 }
 
+function restart_controller_manager() {
+  echo -n "￮ restarting controller manager "
+
+  kubectl rollout restart deployments/operator-controller-manager >/dev/null
+
+  echo "✓"
+}
+
 function resize_nodegroups() {
   if [ -z "$CORTEX_NODEGROUP_NAMES_TO_SCALE" ]; then
     return

manager/manifests/activator.yaml.j2

@@ -81,7 +81,6 @@ spec:
             - "--port=8000"
             - "--autoscaler-url=http://autoscaler.default:8000"
             - "--namespace=default"
-            - "--cluster-config=/configs/cluster/cluster.yaml"
           ports:
             - name: http
               containerPort: 8000
@@ -111,15 +110,6 @@ spec:
           envFrom:
           - configMapRef:
               name: env-vars
-          volumeMounts:
-          - mountPath: /configs/cluster/cluster.yaml
-            name: cluster-config
-            subPath: cluster.yaml
-      volumes:
-      - configMap:
-          defaultMode: 420
-          name: cluster-config
-        name: cluster-config
 ---
 
 apiVersion: v1

manager/manifests/autoscaler.yaml.j2

@@ -84,7 +84,6 @@ spec:
             - "--port=8000"
             - "--prometheus-url=http://prometheus.prometheus:9090"
             - "--namespace=default"
-            - "--cluster-config=/configs/cluster/cluster.yaml"
           ports:
             - containerPort: 8000
           livenessProbe:
@@ -102,16 +101,6 @@ spec:
           envFrom:
           - configMapRef:
               name: env-vars
-          volumeMounts:
-          - mountPath: /configs/cluster/cluster.yaml
-            name: cluster-config
-            subPath: cluster.yaml
-      volumes:
-      - configMap:
-          defaultMode: 420
-          name: cluster-config
-        name: cluster-config
-
 ---
 
 apiVersion: v1

pkg/crds/config/manager/manager.yaml

@@ -11,6 +11,9 @@ spec:
     matchLabels:
       control-plane: controller-manager
   replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
   template:
     metadata:
       labels: