Cr 12025 add cluster wait (#427)

* initial commit

* add-cluster to build dev image

* improved script

* fixed dev build

* fixed script

* updated add-cluster script version

fixed logs

* updated version to 0.0.380

* pr comments

Author: ATGardner

Makefile

@@ -1,4 +1,4 @@
-VERSION=v0.0.379
+VERSION=v0.0.380
 
 OUT_DIR=dist
 YEAR?=$(shell date +"%Y")

build/add-cluster-script.yaml

@@ -58,6 +58,25 @@ steps:
         - name: compare_versions
           on:
             - success
+      branch:
+        only: ["main"]
+
+  build_dev_image:
+    stage: Release
+    type: build
+    title: build and push add-cluster dev image
+    working_directory: manifests/add-cluster
+    image_name: codefresh/csdp-add-cluster
+    tag: ${{CF_BRANCH}}
+    registry: ${{REGISTRY_INTEGRATION_QUAY}}
+    buildkit: true
+    when:
+      steps:
+        - name: compare_versions
+          on:
+            - success
+      branch:
+        ignore: ["main"]
 
   publish_chart:
     stage: Release

cmd/commands/cluster.go

@@ -24,6 +24,7 @@ import (
 	"github.com/codefresh-io/cli-v2/pkg/log"
 	"github.com/codefresh-io/cli-v2/pkg/store"
 	"github.com/codefresh-io/cli-v2/pkg/util"
+	kubeutil "github.com/codefresh-io/cli-v2/pkg/util/kube"
 	kustutil "github.com/codefresh-io/cli-v2/pkg/util/kust"
 
 	"github.com/Masterminds/semver/v3"
@@ -139,15 +140,20 @@ func runClusterAdd(ctx context.Context, opts *ClusterAddOptions) error {
 
 	manifests, err := kustutil.BuildKustomization(k)
 	if err != nil {
-		return fmt.Errorf("failed building kustomization:%w", err)
+		return fmt.Errorf("failed building kustomization: %w", err)
 	}
 
 	if opts.dryRun {
 		fmt.Println(string(manifests))
 		return nil
 	}
 
-	return opts.kubeFactory.Apply(ctx, manifests)
+	err = opts.kubeFactory.Apply(ctx, manifests)
+	if err != nil {
+		return fmt.Errorf("failed applying manifests to cluster: %w", err)
+	}
+
+	return kubeutil.WaitForJob(ctx, opts.kubeFactory, "kube-system", store.Get().AddClusterJobName)
 }
 
 func createAddClusterKustomization(ingressUrl, contextName, server, csdpToken, version string) *kusttypes.Kustomization {
@@ -376,7 +382,7 @@ func runCreateArgoRollouts(ctx context.Context, opts *ClusterCreateArgoRolloutsO
 	err = appProxy.AppProxyClusters().CreateArgoRollouts(ctx, opts.server, opts.namespace)
 	if err != nil {
 		return fmt.Errorf("failed to create argo-rollouts on \"%s'\": %w", opts.server, err)
-	} 
+	}
 
 	log.G(ctx).Infof("created argo-rollouts component on \"%s\"", opts.server)
 

docs/releases/release_notes.md

@@ -23,7 +23,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.379/cf-linux-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.380/cf-linux-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-linux-amd64 /usr/local/bin/cf
@@ -36,7 +36,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.379/cf-darwin-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.380/cf-darwin-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-darwin-amd64 /usr/local/bin/cf

manifests/add-cluster/add-cluster.sh

@@ -36,11 +36,17 @@ STATUS_CODE=$(curl -X POST ${INGRESS_URL}/app-proxy/api/clusters \
   -H 'Content-Type: application/json' \
   -H 'Authorization: '${CSDP_TOKEN}'' \
   -d '{ "name": "'${CONTEXT_NAME}'", "kubeConfig": "'${KUBE_CONFIG_B64}'" }' \
-  -s -o response -w "%{http_code}")
-cat response
+  -skL -o response -w "%{http_code}")
 echo "STATUS_CODE: ${STATUS_CODE}"
+cat response
+echo
+
+if [[ $STATUS_CODE == 000 ]]; then
+  echo "error sending request to runtime"
+  exit 1
+fi
 
-if [ $STATUS_CODE -ge 300 ]; then
+if [[ $STATUS_CODE -ge 300 ]]; then
   echo "error creating cluster in runtime"
   exit $STATUS_CODE
 fi

manifests/add-cluster/helm/Chart.yaml

@@ -6,4 +6,4 @@ type: application
 
 version: 0.2.0
 
-appVersion: 0.2.0
+appVersion: 0.3.0

manifests/add-cluster/kustomize/kustomization.yaml

@@ -4,7 +4,7 @@ namespace: kube-system
 
 images:
   - name: quay.io/codefresh/csdp-add-cluster
-    newTag: 0.2.0
+    newTag: 0.3.0
 
 resources:
   - configmap.yaml

manifests/runtime.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: "{{ namespace }}"
 spec:
   defVersion: 1.0.1
-  version: 0.0.379
+  version: 0.0.380
   bootstrapSpecifier: github.com/codefresh-io/cli-v2/manifests/argo-cd
   components:
     - name: events

pkg/store/store.go

@@ -45,6 +45,7 @@ type Version struct {
 }
 
 type Store struct {
+	AddClusterJobName                   string
 	ArgoCDServerName                    string
 	ArgoCDTokenKey                      string
 	ArgoCDTokenSecret                   string
@@ -151,6 +152,7 @@ func Get() *Store {
 }
 
 func init() {
+	s.AddClusterJobName = "csdp-add-cluster-job"
 	s.ArgoCDServerName = "argocd-server"
 	s.ArgoCDTokenKey = "token"
 	s.ArgoCDTokenSecret = "argocd-token"

pkg/util/kube/kube.go

@@ -216,6 +216,66 @@ func GetClusterSecret(ctx context.Context, kubeFactory kube.Factory, namespace s
 	return res, nil
 }
 
+func WaitForJob(ctx context.Context, f kube.Factory, ns, jobName string) error {
+	var attempt int32
+	return f.Wait(ctx, &kube.WaitOptions{
+		Interval: time.Second * 10,
+		Timeout:  time.Minute * 11, // BackOffLimit of 6 is a total of 630s, or 10m30s
+		Resources: []kube.Resource{
+			{
+				Name:      jobName,
+				Namespace: ns,
+				WaitFunc: func(ctx context.Context, f kube.Factory, ns, name string) (bool, error) {
+					cs, err := f.KubernetesClientSet()
+					if err != nil {
+						return false, err
+					}
+
+					j, err := cs.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{})
+					if err != nil {
+						return false, err
+					}
+
+					if j.Status.Failed > attempt {
+						attempt = j.Status.Failed
+						log.G(ctx).Warnf("Attempt #%d/%d failed:", attempt, *j.Spec.BackoffLimit)
+						printJobLogs(ctx, cs, j)
+					} else if j.Status.Succeeded == 1 {
+						attempt += 1
+						log.G(ctx).Infof("Attempt #%d/%d succeeded:", attempt, *j.Spec.BackoffLimit)
+						printJobLogs(ctx, cs, j)
+					}
+
+					for _, cond := range j.Status.Conditions {
+						if cond.Type == batchv1.JobFailed {
+							err = fmt.Errorf("add-cluster-job failed after %d attempts", j.Status.Failed)
+							break
+						}
+					}
+
+					return j.Status.Succeeded == 1 || j.Status.Failed == *j.Spec.BackoffLimit, err
+				},
+			},
+		},
+	})
+}
+
+func printJobLogs(ctx context.Context, client kubernetes.Interface, job *batchv1.Job) {
+	p, err := getPodByJob(ctx, client, job)
+	if err != nil {
+		log.G(ctx).Errorf("Failed getting pod for job: $s", err.Error())
+		return
+	}
+
+	logs, err := getPodLogs(ctx, client, p.GetNamespace(), p.GetName())
+	if err != nil {
+		log.G(ctx).Errorf("Failed getting logs for pod: $s", err.Error())
+		return
+	}
+
+	fmt.Printf("=====\n%s\n=====\n\n", logs)
+}
+
 func runNetworkTest(ctx context.Context, kubeFactory kube.Factory, urls ...string) error {
 	const networkTestsTimeout = 120 * time.Second
 
@@ -426,8 +486,8 @@ func deleteJob(ctx context.Context, client kubernetes.Interface, job *batchv1.Jo
 }
 
 func getPodByJob(ctx context.Context, client kubernetes.Interface, job *batchv1.Job) (*v1.Pod, error) {
-	pods, err := client.CoreV1().Pods(store.Get().DefaultNamespace).List(ctx, metav1.ListOptions{
-		LabelSelector: "controller-uid=" + job.GetLabels()["controller-uid"],
+	pods, err := client.CoreV1().Pods(job.GetNamespace()).List(ctx, metav1.ListOptions{
+		LabelSelector: "controller-uid=" + job.Spec.Selector.MatchLabels["controller-uid"],
 	})
 	if err != nil {
 		return nil, err