Merge branch '388-ci-aws-marketplace' into 'master'

feat(cf): add CI rules to run AWS Marketplace jobs (#388)

Closes #388

See merge request postgres-ai/database-lab!553

Author: agneum

.gitignore

@@ -12,3 +12,4 @@ engine/bin/
 /engine/configs/run_ci.yaml
 /engine/configs/ci_checker.yml
 /packer/example.com.key
+cloudformation/output/*.yaml

.gitlab-ci.yml

@@ -13,6 +13,7 @@ workflow:
 include:
   - local: 'engine/.gitlab-ci.yml'
   - local: 'ui/.gitlab-ci.yml'
+  - local: 'cloudformation/.gitlab-ci.yml'
 
 empty-job:
   stage: test
@@ -25,4 +26,5 @@ empty-job:
       changes:
         - engine/**/*
         - ui/**/*
+        - cloudformation/**/*
       when: never

cloudformation/.gitlab-ci.yml

@@ -0,0 +1,38 @@
+build_aws_ami:
+  rules:
+    - if: $CI_COMMIT_TAG =~ /^cf\/[0-9.]+[\-_]*[a-zA-Z]+[a-zA-Z0-9.\-_]*[a-zA-Z0-9]+$/
+  stage: build
+  image: ubuntu:20.04
+  before_script:
+    - export CF_VERSION=$(echo ${CI_COMMIT_TAG#"cf/"})
+    - export DLE_VERSION=$(echo ${CF_VERSION%_preview*})
+    - export AWS_DEFAULT_REGION="us-east-1"
+
+  script:
+    - apt-get update -y && apt-get install -y curl jq unzip gnupg2 lsb-release software-properties-common
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+    - add-apt-repository ppa:rmescandon/yq
+    - apt-get update && apt-get install packer yq
+
+    - cd packer
+    - export PKR_VAR_dle_version="v${DLE_VERSION}"
+    - export PKR_VAR_ami_name_prefix="DBLABserver"
+    - packer build template.json.pkr.hcl
+
+    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    - unzip awscliv2.zip
+    - ./aws/install
+
+    - cd ../cloudformation
+    - export DLE_CF_TEMPLATE_FILE="output/dle_cf-${CF_VERSION}.yaml"
+    - sed -e "s|\$DLE_DOCKER_IMAGE_VERSION|$DLE_VERSION|" dle_cf_template.yaml > ${DLE_CF_TEMPLATE_FILE}
+    - bash getAMIs.sh
+
+    - export PACKAGE_NAME="cloudformation"
+    - 'curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file ${DLE_CF_TEMPLATE_FILE} "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${CF_VERSION}/dle_cf-${CF_VERSION}.yaml"'
+
+  artifacts:
+    paths:
+      - cloudformation/output/*.yaml

cloudformation/dle_cf_template.yaml

@@ -285,7 +285,7 @@ Conditions:
   CreateSubDomain:
     !Not [!Equals [!Ref CertificateHostedZone, '']]
   NotCreateSubDomain:
-    !Not [Condition: CreateSubDomain]  
+    !Not [Condition: CreateSubDomain]
 
 Resources:
   LambdaExecutionRole:
@@ -427,7 +427,7 @@ Resources:
             --volume $postgres_conf_path:/home/dblab/standard/postgres/control \
             --env DOCKER_API_VERSION=1.39 \
             --restart always \
-            registry.gitlab.com/postgres-ai/database-lab/dblab-server:3.1.1
+            registry.gitlab.com/postgres-ai/database-lab/dblab-server:$DLE_DOCKER_IMAGE_VERSION
 
           if [ ! -z "${CertificateHostedZone}" ]; then
             export DOMAIN=${CertificateSubdomain}.${CertificateHostedZone}

cloudformation/getAMIs.sh

@@ -10,15 +10,21 @@
 # The script can be run with normal OS user privileges.
 # The script is not supposed to modify anything.
 # There is no warranty. Please check the script upfront. You will use it on your own risk
+
 # String to be used when no AMI is available in region
 NOAMI="NOT_SUPPORTED"
-# Change your aws prfile if needed here:
-PROFILE=" --profile default"
+
+# Change your aws profile if needed here (example, " --profile default"):
+PROFILE=${DLE_AWS_PROFILE:-""}
+
+DLE_CF_TEMPLATE_FILE="${DLE_CF_TEMPLATE_FILE:-dle_cf.yaml}"
+
 # Check whether AWS CLI is installed and in search path
 if ! aws_loc="$(type -p "aws")" || [ -z "$aws_loc" ]; then
-echo "Error: Script requeres AWS CLI . Install it and retry"
+echo "Error: Script requires AWS CLI . Install it and retry"
 exit 1
 fi
+
 # Check whether parameter has been provided
 if [ -z "$1" ]
 then
@@ -29,14 +35,22 @@ NAME=$1
 fi
 echo "Will search for AMIs with name: ${NAME}"
 echo "---------------------------------------"
+
+# Clean AWSRegionArch2AMI list
+yq e -i 'del( .Mappings.AWSRegionArch2AMI )' ${DLE_CF_TEMPLATE_FILE}
+
 ##NAME=DBLABserver*
-R=$(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text ${PROFILE})
-for i in $R; do
-AMI=`aws ec2 describe-images --owners 005923036815 --region $i --filters "Name=name,Values=${NAME}" --output json | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
-if [ -z "$AMI" ]
-then
-AMI=$NOAMI
-fi
-echo  "    "${i}: $'\n' "     "HVM64: ${AMI}
+Regions=$(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text ${PROFILE})
+for i in $Regions; do
+  AMI=`aws ec2 describe-images --owners 005923036815 --region $i --filters "Name=name,Values=${NAME}" --output json | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
+  if [ -z "$AMI" ]; then
+    AMI=$NOAMI
+  fi
+
+  echo  "    "${i}: $'\n' "     "HVM64: ${AMI}
+
+  if [ "$AMI" != "$NOAMI" ]; then
+    yq e -i ".Mappings.AWSRegionArch2AMI.\"${i}\" = {\"HVM64\": \"${AMI}\"}" ${DLE_CF_TEMPLATE_FILE}
+  fi
 done
 

cloudformation/output/.gitkeep

@@ -21,15 +21,15 @@ data "amazon-ami" "base" {
 }
 
 source "amazon-ebs" "base" {
-  ami_description = "Ubuntu 20.04 with  ZFS, Docker, Envoy proxy and Database Lab Engine 3.1 with client CLI."
+  ami_description = "Ubuntu 20.04 with ZFS, Docker, Envoy proxy and Database Lab Engine with client CLI."
   ami_name        = "${var.ami_name_prefix}-${var.dle_version}-${formatdate("YYYY-MM-DD", timestamp())}-${uuidv4()}"
   instance_type   = "t2.large"
   source_ami      = "${data.amazon-ami.base.id}"
   ssh_username    = "ubuntu"
   ami_groups      = ["all"] # This makes our AMI public
   ami_regions     = ["us-east-1","us-east-2","us-west-1","us-west-2","eu-central-1","eu-north-1","ap-south-1","eu-west-3",
                       "eu-west-2","eu-west-1","ap-northeast-3","ap-northeast-2","ap-northeast-1","sa-east-1","ca-central-1",
-                    "ap-southeast-1","ap-southeast-2"] 
+                    "ap-southeast-1","ap-southeast-2"]
 }
 
 build {