Wait for resources to be deleted on cluster update

(cherry picked from commit 5566e6a)

Author: deliahu

manager/install.sh

@@ -122,21 +122,25 @@ function main() {
   echo "✓ configured autoscaling"
 
   kubectl -n=cortex delete --ignore-not-found=true daemonset fluentd >/dev/null 2>&1  # Pods in DaemonSets cannot be modified
+  until [ "$(kubectl -n=cortex get pods -l app=fluentd -o json | jq -j '.items | length')" -eq "0" ]; do sleep 1; done
   envsubst < manifests/fluentd.yaml | kubectl apply -f - >/dev/null
   echo "✓ configured logging"
 
   kubectl -n=cortex delete --ignore-not-found=true daemonset cloudwatch-agent-statsd >/dev/null 2>&1  # Pods in DaemonSets cannot be modified
+  until [ "$(kubectl -n=cortex get pods -l name=cloudwatch-agent-statsd -o json | jq -j '.items | length')" -eq "0" ]; do sleep 1; done
   envsubst < manifests/metrics-server.yaml | kubectl apply -f - >/dev/null
   envsubst < manifests/statsd.yaml | kubectl apply -f - >/dev/null
   echo "✓ configured metrics"
 
   if [[ "$CORTEX_INSTANCE_TYPE" == p* ]] || [[ "$CORTEX_INSTANCE_TYPE" == g* ]]; then
     kubectl -n=cortex delete --ignore-not-found=true daemonset nvidia-device-plugin-daemonset >/dev/null 2>&1  # Pods in DaemonSets cannot be modified
+    until [ "$(kubectl -n=kube-system get pods -l name=nvidia-device-plugin-ds -o json | jq -j '.items | length')" -eq "0" ]; do sleep 1; done
     envsubst < manifests/nvidia.yaml | kubectl apply -f - >/dev/null
     echo "✓ configured gpu support"
   fi
 
-  kubectl -n=cortex delete --ignore-not-found=true deployment operator >/dev/null 2>&1
+  kubectl -n=cortex delete --ignore-not-found=true --grace-period=10 deployment operator >/dev/null 2>&1
+  until [ "$(kubectl -n=cortex get pods -l workloadID=operator -o json | jq -j '.items | length')" -eq "0" ]; do sleep 1; done
   envsubst < manifests/operator.yaml | kubectl apply -f - >/dev/null
   echo "✓ started operator"