Make layer publishing idempotent (#118)

Author: tianchu

CONTRIBUTING.md

@@ -22,8 +22,8 @@ We love pull requests. Here's a quick guide.
      ./scripts/build_layers.sh
 
      # Publish the a testing layer to your own AWS account, and the ARN will be returned
-     # Example: ./scripts/publish_layers.sh us-east-1 Datadog-Python37
-     ./scripts/publish_layers.sh <AWS_REGION> <Layer_Name>
+     # Example: VERSION=1 REGIONS=us-east-1 LAYERS=Datadog-Python37 ./scripts/publish_layers.sh
+     VERSION=<VERSION> REGIONS=<REGION> LAYERS=<LAYER> ./scripts/publish_layers.sh
      ```
 
 1. Ensure the unit tests pass (install Docker if you haven't):

Dockerfile

@@ -16,4 +16,8 @@ RUN find ./python/lib/$runtime/site-packages -name \*.pyc -delete
 
 # Remove botocore (40MB) to reduce package size. aws-xray-sdk
 # installs it, while it's already provided by the Lambda Runtime.
-RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
+RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
+
+# Remove profiling (7MB) to reduce package size.
+# Continous profiling is not yet supported anyway.
+RUN rm -rf ./python/lib/$runtime/site-packages/ddtrace/profiling

scripts/publish_layers.sh

@@ -6,16 +6,16 @@
 # Copyright 2019 Datadog, Inc.
 
 # Publish the datadog python lambda layer across regions, using the AWS CLI
-# Usage: publish_layer.sh [region] [layer]
-# Specifying the region and layer arg will publish the specified layer to the specified region
+# Usage: VERSION=5 REGIONS=us-east-1 LAYERS=Datadog-Python27 publish_layers.sh
+# VERSION is required.
 set -e
 
 # Makes sure any subprocesses will be terminated with this process
 trap "pkill -P $$; exit 1;" INT
 
 PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
 LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip")
-AVAILABLE_LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
+AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 
 # Check that the layer files exist
@@ -27,42 +27,58 @@ do
     fi
 done
 
-# Check region arg
-if [ -z "$1" ]; then
-    echo "Region parameter not specified, running for all available regions."
+# Determine the target regions
+if [ -z "$REGIONS" ]; then
+    echo "Region not specified, running for all available regions."
     REGIONS=$AVAILABLE_REGIONS
 else
-    echo "Region parameter specified: $1"
-    if [[ ! "$AVAILABLE_REGIONS" == *"$1"* ]]; then
-        echo "Could not find $1 in available regions: $AVAILABLE_REGIONS"
+    echo "Region specified: $REGIONS"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then
+        echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS"
         echo ""
         echo "EXITING SCRIPT."
         exit 1
     fi
-    REGIONS=($1)
 fi
 
-echo "Starting publishing layers for regions: $REGIONS"
-
-# Check layer_name arg
-if [ -z "$2" ]; then
-    echo "Layer name parameter not specified, running for all layer names."
-    LAYER_NAMES=("${AVAILABLE_LAYER_NAMES[@]}")
+# Determine the target layers
+if [ -z "$LAYERS" ]; then
+    echo "Layer not specified, running for all layers."
+    LAYERS=("${AVAILABLE_LAYERS[@]}")
 else
-    echo "Layer name parameter specified: $2"
-    if [[ ! " ${AVAILABLE_LAYER_NAMES[@]} " =~ " ${2} " ]]; then
-        echo "Could not find $2 in available layer names: ${AVAILABLE_LAYER_NAMES[@]}"
+    echo "Layer specified: $LAYERS"
+    if [[ ! " ${AVAILABLE_LAYERS[@]} " =~ " ${LAYERS} " ]]; then
+        echo "Could not find $LAYERS in available layers: ${AVAILABLE_LAYERS[@]}"
         echo ""
         echo "EXITING SCRIPT."
         exit 1
     fi
-    LAYER_NAMES=($2)
 fi
 
+# Determine the target layer version
+if [ -z "$VERSION" ]; then
+    echo "Layer version not specified"
+    echo ""
+    echo "EXITING SCRIPT."
+    exit 1
+else
+    echo "Layer version specified: $VERSION"
+fi
 
+read -p "Ready to publish version $VERSION of layers ${LAYERS[*]} to regions ${REGIONS[*]} (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+    echo "Exiting"
+    exit 1
+fi
 
-
-echo "Publishing layers: ${LAYER_NAMES[*]}"
+index_of_layer() {
+    layer_name=$1
+    for i in "${!AVAILABLE_LAYERS[@]}"; do
+        if [[ "${AVAILABLE_LAYERS[$i]}" = "${layer_name}" ]]; then
+            echo "${i}";
+        fi
+    done
+}
 
 publish_layer() {
     region=$1
@@ -76,46 +92,50 @@ publish_layer() {
         --compatible-runtimes $aws_version_key \
                         | jq -r '.Version')
 
-    aws lambda add-layer-version-permission --layer-name $layer_name \
+    permission=$(aws lambda add-layer-version-permission --layer-name $layer_name \
         --version-number $version_nbr \
         --statement-id "release-$version_nbr" \
         --action lambda:GetLayerVersion --principal "*" \
-        --region $region
+        --region $region)
 
-    echo "Published layer for region $region, python version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
-}
-
-BATCH_SIZE=1
-PIDS=()
-
-wait_for_processes() {
-    for pid in "${PIDS[@]}"; do
-        wait $pid
-    done
-    PIDS=()
+    echo $version_nbr
 }
 
 for region in $REGIONS
 do
     echo "Starting publishing layer for region $region..."
-
     # Publish the layers for each version of python
-    i=0
-    for layer_name in "${LAYER_NAMES[@]}"; do
-        aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$i]}"
-        layer_path="${LAYER_PATHS[$i]}"
-
-        publish_layer $region $layer_name $aws_version_key $layer_path &
-        PIDS+=($!)
-        if [ ${#PIDS[@]} -eq $BATCH_SIZE ]; then
-            wait_for_processes
+    for layer_name in "${LAYERS[@]}"; do
+        latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`')
+        if [ $latest_version -ge $VERSION ]; then
+            echo "Layer $layer_name version $VERSION already exists in region $region, skipping..."
+            continue
+        elif [ $latest_version -lt $((VERSION-1)) ]; then
+            read -p "WARNING: The latest version of layer $layer_name in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT
+            if [ "$CONT" != "y" ]; then
+                echo "Exiting"
+                exit 1
+            fi
         fi
 
-        i=$(expr $i + 1)
-
+        index=$(index_of_layer $layer_name)
+        aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}"
+        layer_path="${LAYER_PATHS[$index]}"
+
+        while [ $latest_version -lt $VERSION ]; do
+            latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path)
+            echo "Published version $latest_version for layer $layer_name in region $region"
+
+            # This shouldn't happen unless someone manually deleted the latest version, say 28
+            # and then try to republish it again. The published version is actually be 29, because
+            # Lambda layers are immutable and AWS will skip deleted version and use the next number. 
+            if [ $latest_version -gt $VERSION ]; then
+                echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!"
+                echo "Exiting"
+                exit 1
+            fi
+        done
     done
 done
 
-wait_for_processes
-
 echo "Done !"

scripts/publish_prod.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Use with `aws-vault exec <PROFILE> -- ./publish_prod.sh <DESIRED_NEW_VERSION>
+# Use with `./publish_prod.sh <DESIRED_NEW_VERSION>
 
 set -e
 
@@ -14,8 +14,8 @@ else
     git pull origin main
 fi
 
-# # Ensure no uncommitted changes
-if [ -n "$(git status --porcelain)" ]; then 
+# Ensure no uncommitted changes
+if [ -n "$(git status --porcelain)" ]; then
     echo "Detected uncommitted changes, aborting"
     exit 1
 fi
@@ -37,44 +37,27 @@ AWS_PROFILE=govcloud-us1-fed-human-engineering aws sts get-caller-identity
 aws-vault exec prod-engineering -- aws sts get-caller-identity
 
 # Ensure pypi registry access
-read -p "Do you have the PyPi login credentials for datadog account (y/n)? " -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
-    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
-fi
-
-echo 'Checking existing layers in commercial AWS regions'
-aws-vault exec prod-engineering -- ./scripts/list_layers.sh
-
-echo 'Checking existing layers in GovCloud AWS regions'
-saml2aws login -a govcloud-us1-fed-human-engineering
-AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh
-
-read -p "Do the layer lists look good? Proceed publishing the new version (y/n)? " -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
-    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
+read -p "Do you have the PyPi login credentials for datadog account (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+    echo "Exiting"
+    exit 1
 fi
 
 VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./datadog_lambda/__init__.py)
 CURRENT_VERSION=$(echo "$VERSION_LINE" | cut -d '"' -f 2)
+LAYER_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2)
 
-read -p "Ready to publish layers and update the version from $CURRENT_VERSION to $NEW_VERSION? (y/n)" -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
-    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
+read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+    echo "Exiting"
+    exit 1
 fi
 
 echo
 echo "Replacing __version__ in ./datadog_lambda/__init__.py"
 echo
 sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"$NEW_VERSION\"/" ./datadog_lambda/__init__.py
 
-git commit ./datadog_lambda/__init__.py -m "Update module version to ${NEW_VERSION}"
-
 echo
 echo "Building layers..."
 ./scripts/build_layers.sh
@@ -85,25 +68,16 @@ aws-vault exec prod-engineering -- ./scripts/sign_layers.sh prod
 
 echo
 echo "Publishing layers to commercial AWS regions"
-aws-vault exec prod-engineering -- ./scripts/publish_layers.sh
+VERSION=$LAYER_VERSION aws-vault exec prod-engineering -- ./scripts/publish_layers.sh
 
 echo "Publishing layers to GovCloud AWS regions"
 saml2aws login -a govcloud-us1-fed-human-engineering
-AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh
-
-echo 'Checking published layers in commercial AWS regions'
-aws-vault exec prod-engineering -- ./scripts/list_layers.sh
-
-echo 'Checking published layers in GovCloud AWS regions'
-saml2aws login -a govcloud-us1-fed-human-engineering
-AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh
+VERSION=$LAYER_VERSION AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh
 
-
-read -p "Do the layer lists look good? Ready to publish $NEW_VERSION to Pypi? (y/n)" -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
-    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
+read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+    echo "Exiting"
+    exit 1
 fi
 
 echo
@@ -112,11 +86,11 @@ echo "Publishing to https://pypi.org/project/datadog-lambda/"
 
 echo
 echo 'Publishing updates to github'
-MINOR_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2)
+git commit ./datadog_lambda/__init__.py -m "Update module version to ${NEW_VERSION}"
 git push origin main
-git tag "v$MINOR_VERSION"
-git push origin "refs/tags/v$MINOR_VERSION"
+git tag "v$LAYER_VERSION"
+git push origin "refs/tags/v$LAYER_VERSION"
 
 echo
-echo "Now create a new release with the tag v${MINOR_VERSION} created"
-echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$MINOR_VERSION&title=v$MINOR_VERSION"
+echo "Now create a new release with the tag v${LAYER_VERSION} created"
+echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION"

scripts/publish_sandbox.sh

@@ -2,5 +2,5 @@
 set -e
 
 ./scripts/build_layers.sh
-./scripts/sign_layers.sh sandbox
-./scripts/publish_layers.sh sa-east-1
+aws-vault exec sandbox-account-admin -- ./scripts/sign_layers.sh sandbox
+aws-vault exec sandbox-account-admin -- ./scripts/publish_layers.sh