adjust appwrapper controller startup

Author: dgrove-oss

.github/workflows/olm_tests.yaml

@@ -54,6 +54,10 @@ jobs:
       - name: Setup and start KinD cluster
         uses: ./common/github-actions/kind
 
+      - name: Deploy Kueue
+        run: |
+          make kueue-e2e
+
       - name: Install OLM
         run: |
           kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml

config/rbac/role.yaml

@@ -61,6 +61,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - apps
   resources:

go.mod

@@ -97,8 +97,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.29.0 // indirect
-	k8s.io/apiserver v0.29.1 // indirect
+	k8s.io/apiserver v0.29.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	sigs.k8s.io/jobset v0.3.1 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -494,13 +494,13 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ=
 k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A=
 k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0=
-k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0=
-k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc=
+k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg=
+k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8=
 k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
 k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8=
 k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
-k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g=
-k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw=
+k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ=
+k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ=
 k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg=
 k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA=
 k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=

main.go

@@ -188,14 +188,17 @@ func main() {
 	}
 
 	setupLog.Info("setting up indexers")
-	exitOnError(setupIndexers(ctx, mgr, kubeClient, cfg), "unable to setup indexers")
+	exitOnError(setupIndexers(ctx, mgr, cfg), "unable to setup indexers")
 
-        setupLog.Info("setting up health endpoints")
+	setupLog.Info("setting up health endpoints")
 	exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check")
 
 	setupLog.Info("setting up RayCluster controller")
 	go waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady)
 
+	setupLog.Info("setting up AppWrapper controller")
+	go setupAppWrapperController(mgr, cfg, certsReady)
+
 	setupLog.Info("starting manager")
 	exitOnError(mgr.Start(ctx), "error running manager")
 }
@@ -266,31 +269,25 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage
 			}
 		}
 	}
+}
+
+func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
+	setupLog.Info("Waiting for certificate generation to complete")
+	<-certsReady
+	setupLog.Info("Certs ready")
 
 	if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) {
-		hasAW, errAW := hasAPIResourceForGVK(dc, awv1beta2.GroupVersion.WithKind("AppWrapper"))
-		hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload"))
-		if hasAW && hasWL {
-			exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook")
-			exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller")
-		} else if errAW != nil || errWL != nil {
-			exitOnError(err, "Could not determine if Workload and AppWrapper CRDs present on cluster.")
-		} else {
-			setupLog.Info("AppWrapper controller disabled", "Workload CRD present", hasWL,
-				"AppWrapper CRD present", hasAW, "Config flag value", *cfg.AppWrapper.Enabled)
-		}
+		exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook")
+		exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller")
+	} else {
+		setupLog.Info("AppWrapper controller disabled", "Config flag value", *cfg.AppWrapper.Enabled)
 	}
 }
 
-func setupIndexers(ctx context.Context, mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration) error {
+func setupIndexers(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration) error {
 	if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) {
-		hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload"))
-		if hasWL {
-			if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil {
-				return fmt.Errorf("workload indexer: %w", err)
-			}
-		} else if errWL != nil {
-			return fmt.Errorf("checking Workload CR: %w", errWL)
+		if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil {
+			return fmt.Errorf("workload indexer: %w", err)
 		}
 	}
 	return nil