Add docs and error message for allowing other IAM users to access cluster commands (#1392)

Author: deliahu

docs/miscellaneous/security.md

@@ -76,3 +76,33 @@ It is possible to further restrict access by limiting access to particular resou
 ### CLI
 
 In order to connect to the operator via the CLI, you must provide valid AWS credentials for any user with access to the account. No special permissions are required. The CLI can be configured using the `cortex env configure ENVIRONMENT_NAME` command (e.g. `cortex env configure aws`).
+
+### Running `cortex cluster` commands from different IAM users
+
+By default, the `cortex cluster *` commands can only be executed by the IAM user who created the Cortex cluster. To grant access to additional IAM users, follow these steps:
+
+1. Install `eksctl` by following these [instructions](https://eksctl.io/introduction/#installation).
+
+1. Determine the ARN of the IAM user that you would like to grant access to. You can get the ARN via the [IAM dashboard](https://console.aws.amazon.com/iam/home#/users), or by running `aws iam get-user` on a machine that is authenticated as the IAM user (or `AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** aws iam get-user` on any machine, using the credentials of the IAM user). The ARN should look similar to `arn:aws:iam::764403040417:user/my-username`.
+
+1. Set the following environment variables:
+
+    ```bash
+    CREATOR_AWS_ACCESS_KEY_ID=***      # access key ID for the IAM user that created the cluster
+    CREATOR_AWS_SECRET_ACCESS_KEY=***  # secret access key for the IAM user that created the cluster
+    NEW_USER_ARN=***                   # ARN of the IAM user to grant access to
+    CLUSTER_NAME=***                   # the name of your cortex cluster (will be "cortex" unless you specified a different name in your cluster configuration file)
+    CLUSTER_REGION=***                 # the region of your cortex cluster
+    ```
+
+1. Run the following command:
+
+    ```bash
+    AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl create iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --group system:masters --username $NEW_USER_ARN
+    ```
+
+1. To revoke access in the future, run:
+
+    ```bash
+    AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl delete iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --all
+    ```

manager/debug.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORTEX_VERSION_MINOR=master
+
 debug_out_path="$1"
 mkdir -p "$(dirname "$debug_out_path")"
 
@@ -23,6 +25,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
 fi
 
 eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/miscellaneous/security#running-cortex-cluster-commands-from-different-iam-users"; exit 1; fi
 
 echo -n "gathering cluster data"
 

manager/info.sh

@@ -16,6 +16,8 @@
 
 set -eo pipefail
 
+CORTEX_VERSION_MINOR=master
+
 function get_operator_endpoint() {
   kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/'
 }
@@ -34,6 +36,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
 fi
 
 eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/miscellaneous/security#running-cortex-cluster-commands-from-different-iam-users"; exit 1; fi
 
 operator_endpoint=$(get_operator_endpoint)
 api_load_balancer_endpoint=$(get_api_load_balancer_endpoint)

manager/install.sh

@@ -17,6 +17,7 @@
 set -eo pipefail
 
 export CORTEX_VERSION=master
+export CORTEX_VERSION_MINOR=master
 EKSCTL_TIMEOUT=45m
 mkdir /workspace
 
@@ -199,6 +200,7 @@ function main() {
   fi
 
   eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+  out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/miscellaneous/security#running-cortex-cluster-commands-from-different-iam-users"; exit 1; fi
 
   # pre-download images on cortex cluster up
   if [ "$arg1" != "--update" ]; then

manager/refresh.sh

@@ -16,6 +16,8 @@
 
 set -e
 
+CORTEX_VERSION_MINOR=master
+
 cluster_config_out_path="$1"
 mkdir -p "$(dirname "$cluster_config_out_path")"
 
@@ -25,6 +27,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
 fi
 
 eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/miscellaneous/security#running-cortex-cluster-commands-from-different-iam-users"; exit 1; fi
 
 kubectl get -n=default configmap cluster-config -o yaml >> cluster_configmap.yaml
 python refresh_cluster_config.py cluster_configmap.yaml tmp_cluster_config.yaml