Update versions (#2398)

Author: deliahu

.circleci/config.yml

@@ -10,8 +10,8 @@ commands:
           name: Install Go
           command: |
             sudo rm -rf /usr/local/go
-            wget https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz
-            sudo tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz
+            wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz
+            sudo tar -C /usr/local -xzf go1.17.3.linux-amd64.tar.gz
             rm -rf go*.tar.gz
             echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
             echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
@@ -94,7 +94,7 @@ jobs:
 
   test:
     machine:
-      image: ubuntu-1604:202104-01  # machine executor necessary to run go integration tests
+      image: ubuntu-1604:202104-01 # machine executor necessary to run go integration tests
     resource_class: medium
     steps:
       - checkout

CONTRIBUTING.md

@@ -22,10 +22,10 @@ To install Go on linux, run:
 
 ```bash
 mkdir -p ~/bin && \
-wget https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz && \
-sudo tar -xvf go1.16.5.linux-amd64.tar.gz && \
+wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \
+sudo tar -xvf go1.17.3.linux-amd64.tar.gz && \
 sudo mv go /usr/local && \
-rm go1.16.5.linux-amd64.tar.gz && \
+rm go1.17.3.linux-amd64.tar.gz && \
 echo 'export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"' >> $HOME/.bashrc
 ```
 

build/generate_ami_mapping.go

@@ -222,7 +222,7 @@ func main() {
 		json.Unmarshal(jsonBytes, &k8sVersionMap)
 	}
 
-	k8sVersion := "1.20"
+	k8sVersion := "1.21"
 
 	if k8sVersionMap[k8sVersion] == nil {
 		k8sVersionMap[k8sVersion] = map[string]map[string]string{}

dev/load.go

@@ -109,7 +109,7 @@ func runConstantRequestsPerSecond(url string, jsonBytes []byte) {
 	ticker := time.NewTicker(_requestInterval)
 	done := make(chan bool)
 
-	c := make(chan os.Signal)
+	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-c
@@ -203,7 +203,7 @@ func runConstantInFlightIteration(url string, jsonBytes []byte, loopNum int) boo
 
 	wasKilled := false
 	killed := make(chan bool)
-	c := make(chan os.Signal)
+	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-c

dev/versions.md

@@ -7,8 +7,8 @@
 1. Update `generate_eks.py` if necessary
 1. Check that `eksctl utils write-kubeconfig` log filter still behaves as desired, and logs in `cortex cluster up` look good.
 1. Update eksctl on your dev
-   machine: `curl --location "https://github.com/weaveworks/eksctl/releases/download/0.51.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin`
-1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.51.0/userdocs/src/usage/minimum-iam-policies.md
+   machine: `curl --location "https://github.com/weaveworks/eksctl/releases/download/v0.67.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin`
+1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.67.0/userdocs/src/usage/minimum-iam-policies.md
 
 ## Kubernetes
 
@@ -36,11 +36,18 @@
 1. Update the version in `images/manager/Dockerfile` and `images/operator/Dockerfile`
 1. Update your local version and alert developers
     * Linux:
-        1. `curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl`
-        1. `chmod +x ./kubectl`
-        1. `sudo mv -f ./kubectl /usr/local/bin/kubectl`
-        1. refresh shell
-        1. `kubectl version`
+
+     ```shell
+        mkdir -p $HOME/temp && \
+        cd $HOME/temp && \
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+        chmod +x ./kubectl && \
+        sudo mv -f ./kubectl /usr/local/bin/kubectl && \
+        if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \
+        cd - && \
+        kubectl version
+     ```
+
     * Mac:
         1. `brew upgrade kubernetes-cli`
         1. refresh shell
@@ -57,33 +64,40 @@
 
 ## AWS CNI
 
-1. Update the CNI version in `eks_cluster.yaml` ([CNI releases](https://github.com/aws/amazon-vpc-cni-k8s/releases))
+1. Update the CNI version in `generate_eks.py` ([CNI releases](https://github.com/aws/amazon-vpc-cni-k8s/releases))
 1. Update the go module version (see `Go > Non-versioned modules` section below)
 1. Check if new instance types were added by running the script below (update the two env vars at the top).
    1. If there are new instance types, check if any changes need to be made to `servicequotas.go` or `validateInstanceType()`.
 
 ```bash
-PREV_RELEASE=1.7.10
-NEW_RELEASE=1.8.0
+PREV_RELEASE=1.8.0
+NEW_RELEASE=1.9.3
 wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${PREV_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; wget -q -O cni_supported_instances_new.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${NEW_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; git diff --no-index cni_supported_instances_prev.txt cni_supported_instances_new.txt; rm -rf cni_supported_instances_prev.txt; rm -rf cni_supported_instances_new.txt
 ```
 
 ## Go
 
 1. Find the latest release on Golang's [release page](https://golang.org/doc/devel/release.html) (
    or [downloads page](https://golang.org/dl/)) and check the changelog
-1. Search the codebase for the current minor version (e.g. `1.16`), update versions as appropriate
+1. Search the codebase for the current minor version (e.g. `1.17`), update versions as appropriate
 1. Update your local version and alert developers:
     * Linux:
-        1. `wget https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz`
-        1. `tar -xvf go1.16.5.linux-amd64.tar.gz`
-        1. `sudo rm -rf /usr/local/go`
-        1. `sudo mv -f go /usr/local`
-        1. `rm go1.16.5.linux-amd64.tar.gz`
-        1. refresh shell
-        1. `go version`
+
+     ```shell
+        mkdir -p $HOME/temp
+        cd $HOME/temp
+        wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \
+        tar -xvf go1.17.3.linux-amd64.tar.gz && \
+        sudo rm -rf /usr/local/go && \
+        sudo mv -f go /usr/local && \
+        rm go1.17.3.linux-amd64.tar.gz && \
+        if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \
+        cd - && \
+        go version
+     ```
+
     * Mac:
-        1. `brew upgrade go` or `brew install go@1.16`
+        1. `brew upgrade go` or `brew install go@1.17`
         1. refresh shell
         1. `go version`
 1. Update go modules as necessary
@@ -92,7 +106,7 @@ wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aw
 
 ### Kubernetes client
 
-1. Find the latest patch release for the minor kubernetes version that we use (e.g. for k8s 1.20, use `client-go` version `v0.20.X`, where `X` is the latest available patch release)
+1. Find the latest patch [release](https://github.com/kubernetes/client-go) for the minor kubernetes version that we use (e.g. for k8s 1.21, use `client-go` version `v0.21.X`, where `X` is the latest available patch release)
 1. Follow the "Update non-versioned modules" instructions using the updated version for `k8s.io/client-go`
 
 ### Istio client
@@ -102,7 +116,7 @@ wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aw
 
 ### docker/engine/client
 
-1. Find the latest tag from [releases](https://github.com/docker/engine/releases)
+1. Find the latest tag from [here](https://github.com/docker/engine/tags)
 1. Follow the "Update non-versioned modules" instructions using the updated version for `docker/engine`
 
 _note: docker client installation may be able to be improved,
@@ -131,21 +145,23 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_
 ### Non-versioned modules
 
 1. `rm -rf go.mod go.sum && go mod init && go clean -modcache`
-1. `go get k8s.io/client-go@v0.20.8 && go get k8s.io/apimachinery@v0.20.8 && go get k8s.io/api@v0.20.8`
-1. `go get istio.io/client-go@1.10.2 && go get istio.io/api@1.10.2`
-1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.8.0`
+1. `go get k8s.io/client-go@v0.21.6 && go get k8s.io/apimachinery@v0.21.6 && go get k8s.io/api@v0.21.6`
+1. `go get istio.io/client-go@v1.11.4 && go get istio.io/api@1.11.4`
+1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.9.3`
 1. `go get github.com/cortexlabs/yaml@31e52ba8433b683c471ef92cf1711fe67671dac5`
 1. `go get github.com/cortexlabs/go-input@8b67a7a7b28d1c45f5c588171b3b50148462b247`
 1. `go get github.com/xlab/treeprint@v1.0.0`
+1. `go get -u sigs.k8s.io/controller-runtime@v0.8.3`
 1. `echo -e '\nreplace github.com/docker/docker => github.com/docker/engine v19.03.13' >> go.mod`
 1. `go get -u github.com/docker/distribution`
 1. `go mod tidy`
-1. For every non-indirect, non-hardcoded dependency in go.mod, update with `go get -u <path>`
-1. `go mod tidy`
-1. Re-run the relevant hardcoded `go get` commands above
-1. `go mod tidy`
-1. `make test`
-1. `go mod tidy`
+1. Potentially skip these steps
+   1. For every non-indirect, non-hardcoded dependency in go.mod, update with `go get -u <path>`
+   1. `go mod tidy`
+   1. Re-run the relevant hardcoded `go get` commands above
+   1. `go mod tidy`
+   1. `make test`
+   1. `go mod tidy`
 1. Check that the diff in `go.mod` is reasonable
 
 ## Nvidia device plugin
@@ -188,9 +204,21 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_
    set the tree to the tag for the chosen release, and open `cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml`
    (e.g. <https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.20.0/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml>)
 1. Resolve merge conflicts with the template in `manager/manifests/cluster-autoscaler.yaml.j2`.
-1. Pull the release branch from the upstream repo to Cortex's fork on [Github](https://github.com/cortexlabs/autoscaler).
-1. Apply the rate-limiter changes from the previous version to the new one (currently sitting on `cluster-autoscaler-release-1.20` branch).
-1. Update `-b` flag's value from `git clone` command in `images/cluster-autoscaler/Dockerfile` to the branch name of the latest release from Cortex's fork.
+1. Clone our fork: `git clone git@github.com:cortexlabs/autoscaler.git`
+1. Checkout our updated branch: `git checkout cluster-autoscaler-1.21.1-cortex`
+1. List the most recent commit: `git log`
+1. Reset the latest commit (use the SHA of the last non-cortex commit): `git reset <SHA>`
+1. `git add *`
+1. `git stash`
+1. `git remote add upstream https://github.com/kubernetes/autoscaler.git`
+1. `git fetch upstream`
+1. Checkout the appropriate version tag, e.g. `git checkout cluster-autoscaler-1.21.1 -b cluster-autoscaler-1.21.1-cortex`
+1. `git stash pop`
+1. Resolve any merge conflicts
+1. Unstage and check the diff
+1. `git commit -am "Add rate limiter"`
+1. `git push origin cluster-autoscaler-1.21.1-cortex`
+1. Update `images/cluster-autoscaler/Dockerfile` to use the new branch name (e.g. "cluster-autoscaler-1.21.1") in the `-b` flag's value from `git clone`.
 1. Match the Go version of the builder in `images/cluster-autoscaler/Dockerfile` with that of the [cluster autoscaler](https://github.com/kubernetes/autoscaler)'s Dockerfile.
 
 ## FluentBit