From 9ebed81afda3127fb9bd42dbf359582c1d4ae185 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 17:29:29 +0200 Subject: [PATCH 1/9] add release-workflow --- .github/workflows/cloudpod_release.yml | 94 ++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .github/workflows/cloudpod_release.yml diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml new file mode 100644 index 0000000..36cb2d4 --- /dev/null +++ b/.github/workflows/cloudpod_release.yml @@ -0,0 +1,94 @@ +on: + workflow_dispatch: + inputs: + release-tag: + type: string + required: true + description: This will be the version of the release, but will also be used as 'tag' for the localstack docker image + push: + paths-ignore: + - 'README.md' + branches: + - main + - cloudpod_workflow + +permissions: + contents: write + +name: Create Release +jobs: + release: + name: Create Release for Cloud Pod + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install CDK + run: | + npm install -g aws-cdk-local aws-cdk + cdklocal --version + + - name: Install dependencies + run: | + yarn + + - name: Start LocalStack + env: + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + DNS_ADDRESS: 0 + run: | + export SQS_QUEUE="sqs-fargate-queue" + export NETWORK_NAME="localstack-shared-net" + docker network create $NETWORK_NAME + pip install localstack awscli-local[ver1] + pip install terraform-local + docker pull localstack/localstack-pro:latest + # Start LocalStack in the background + LS_LOG=trace LAMBDA_DOCKER_NETWORK=$NETWORK_NAME DOCKER_FLAGS="--network $NETWORK_NAME" DEBUG=1 localstack start -d + # Wait 30 seconds for the LocalStack container to become ready before timing out + echo "Waiting for LocalStack startup..." + localstack wait -t 15 + echo "Startup complete" + + - name: Deploy using CDK + run: | + docker build -t go-fargate . + cd cdk + npm install + cdklocal bootstrap aws://000000000000/us-east-1 + cdklocal deploy --require-approval never + + # TODO run tests + + - name: Save the Cloud Pod + uses: HarshCasper/cloud-pod-save@v0.1.0 + with: + name: 'release-pod.zip' + location: 'disk' + + - name: Prepare Release Notes + run: | + echo "This release includes the Cloud Pod of the sample created with LocalStack Version \`${{ inputs.release-tag || 'latest'}}\`." > Release.txt + echo "Please be aware, that your running LocalStack instance need to be started with \`LAMBDA_DOCKER_NETWORK\`and \`DOCKER_FLAGS\`: " >> Release.txt + echo " * the network must have the name 'localstack-shared-net', e.g.: \`NETWORK_NAME=\"localstack-shared-net\"\`" >> Release.txt + echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=$NETWORK_NAME DOCKER_FLAGS="--network $NETWORK_NAME" DEBUG=1 localstack start -d \`" >> Release.txt + echo "You can download the \`release-pod.zip\` and inject it manually by running \`localstack pod load file://release-pod.zip\`, or use the Cloud Pods Launchpad." >> Release.txt + echo "### Cloud Pods Launchpad" >> Release.txt + echo "You can click the Launchpad to inject the the pod into your running LocalStack instance using the WebUI:" >> Release.txt + echo "[![LocalStack Pods Launchpad](https://localstack.cloud/gh/launch-pod-badge.svg)](https://app.localstack.cloud/launchpad?url=https://github.com/$GITHUB_REPOSITORY/releases/download/${{ inputs.release-tag || 'latest'}}/release-pod.zip)" >> Release.txt + + - name: Create Release + id: create_release + uses: softprops/action-gh-release@v1 + with: + tag_name: "${{ inputs.release-tag || 'latest'}}" + name: "Cloud Pod for LocalStack Version '${{ inputs.release-tag || 'latest'}}'" + body_path: ./Release.txt + files: | + ./release-pod.zip From 43973deda500e004f953830b964103993f132cdf Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 17:42:21 +0200 Subject: [PATCH 2/9] fix release text --- .github/workflows/cloudpod_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index 36cb2d4..2c4cf1b 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -77,7 +77,7 @@ jobs: echo "This release includes the Cloud Pod of the sample created with LocalStack Version \`${{ inputs.release-tag || 'latest'}}\`." > Release.txt echo "Please be aware, that your running LocalStack instance need to be started with \`LAMBDA_DOCKER_NETWORK\`and \`DOCKER_FLAGS\`: " >> Release.txt echo " * the network must have the name 'localstack-shared-net', e.g.: \`NETWORK_NAME=\"localstack-shared-net\"\`" >> Release.txt - echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=$NETWORK_NAME DOCKER_FLAGS="--network $NETWORK_NAME" DEBUG=1 localstack start -d \`" >> Release.txt + echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=\$NETWORK_NAME DOCKER_FLAGS="--network \$NETWORK_NAME" DEBUG=1 localstack start -d\` \n" >> Release.txt echo "You can download the \`release-pod.zip\` and inject it manually by running \`localstack pod load file://release-pod.zip\`, or use the Cloud Pods Launchpad." >> Release.txt echo "### Cloud Pods Launchpad" >> Release.txt echo "You can click the Launchpad to inject the the pod into your running LocalStack instance using the WebUI:" >> Release.txt From 26ca67a2d35d346ccc111c4809faa6c6a4202bde Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 18:15:05 +0200 Subject: [PATCH 3/9] fixup --- .github/workflows/cloudpod_release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index 2c4cf1b..58eb67d 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -77,7 +77,8 @@ jobs: echo "This release includes the Cloud Pod of the sample created with LocalStack Version \`${{ inputs.release-tag || 'latest'}}\`." > Release.txt echo "Please be aware, that your running LocalStack instance need to be started with \`LAMBDA_DOCKER_NETWORK\`and \`DOCKER_FLAGS\`: " >> Release.txt echo " * the network must have the name 'localstack-shared-net', e.g.: \`NETWORK_NAME=\"localstack-shared-net\"\`" >> Release.txt - echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=\$NETWORK_NAME DOCKER_FLAGS="--network \$NETWORK_NAME" DEBUG=1 localstack start -d\` \n" >> Release.txt + echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=\$NETWORK_NAME DOCKER_FLAGS="--network \$NETWORK_NAME" DEBUG=1 localstack start -d\`" >> Release.txt + echo "" >> Release.txt echo "You can download the \`release-pod.zip\` and inject it manually by running \`localstack pod load file://release-pod.zip\`, or use the Cloud Pods Launchpad." >> Release.txt echo "### Cloud Pods Launchpad" >> Release.txt echo "You can click the Launchpad to inject the the pod into your running LocalStack instance using the WebUI:" >> Release.txt From f5ba76b070efcca3896470f9e7e7f84870d22409 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 18:31:10 +0200 Subject: [PATCH 4/9] add dummy data to the release pod --- .github/workflows/cloudpod_release.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index 58eb67d..9ac50e3 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -63,9 +63,18 @@ jobs: npm install cdklocal bootstrap aws://000000000000/us-east-1 cdklocal deploy --require-approval never - + # TODO run tests + - name: Add some Dummy Data + run: | + echo "Send message to the sqs queue that should be stored in the db" + awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "hello world"}' + + echo "Giving the container time to write the message, then check if the fargate container successfully wrote the sqs message into the dynamodb table" + sleep 3 + awslocal dynamodb scan --table-name sqs-fargate-ddb-table + - name: Save the Cloud Pod uses: HarshCasper/cloud-pod-save@v0.1.0 with: From 51476bb762ee3da943b7590f0d09dd2f3d19b0f3 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 18:43:15 +0200 Subject: [PATCH 5/9] add missing env to create dummy data --- .github/workflows/cloudpod_release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index 9ac50e3..83769b4 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -69,9 +69,10 @@ jobs: - name: Add some Dummy Data run: | echo "Send message to the sqs queue that should be stored in the db" + SQS_QUEUE="sqs-fargate-queue" awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "hello world"}' - echo "Giving the container time to write the message, then check if the fargate container successfully wrote the sqs message into the dynamodb table" + echo "waiting to process the message..." sleep 3 awslocal dynamodb scan --table-name sqs-fargate-ddb-table @@ -87,6 +88,7 @@ jobs: echo "Please be aware, that your running LocalStack instance need to be started with \`LAMBDA_DOCKER_NETWORK\`and \`DOCKER_FLAGS\`: " >> Release.txt echo " * the network must have the name 'localstack-shared-net', e.g.: \`NETWORK_NAME=\"localstack-shared-net\"\`" >> Release.txt echo " * start LocalStack using: \`docker network create $NETWORK_NAME 2> /dev/null; LAMBDA_DOCKER_NETWORK=\$NETWORK_NAME DOCKER_FLAGS="--network \$NETWORK_NAME" DEBUG=1 localstack start -d\`" >> Release.txt + echo " * use the SQS_QUEUE=\"sqs-fargate-queue\" to send further messages that will be written to DynamoDb" >> Release.txt echo "" >> Release.txt echo "You can download the \`release-pod.zip\` and inject it manually by running \`localstack pod load file://release-pod.zip\`, or use the Cloud Pods Launchpad." >> Release.txt echo "### Cloud Pods Launchpad" >> Release.txt From b8506753cece9b17ea01ec1f1d79e735f5d060f3 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 18:54:12 +0200 Subject: [PATCH 6/9] add test_cloudpods workflow --- .github/workflows/cloudpod_release.yml | 1 - .github/workflows/test_cloudpods.yml | 126 +++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test_cloudpods.yml diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index 83769b4..bc0dcd5 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -10,7 +10,6 @@ on: - 'README.md' branches: - main - - cloudpod_workflow permissions: contents: write diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml new file mode 100644 index 0000000..902e94d --- /dev/null +++ b/.github/workflows/test_cloudpods.yml @@ -0,0 +1,126 @@ +name: Test Released Cloud Pods + +on: + schedule: + # “At 00:00 on Saturday.” + - cron: "0 0 * * 6" + workflow_dispatch: + push: # TODO remove just for testing + paths-ignore: + - 'README.md' + branches: + - cloudpod_workflow +permissions: + contents: write + +jobs: + get-releases: + name: Retrieve Released Cloud Pods + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + output=$(gh api repos/$GITHUB_REPOSITORY/releases | jq -r '[.[].tag_name]') + output=$(echo $output | tr '\n' ' ') + echo "matrix=$output" >> $GITHUB_OUTPUT + + test-pod-release: + needs: get-releases + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + tag: ${{ fromJson(needs.get-releases.outputs.matrix) }} + steps: + - name: Retrieve Pod + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # TODO the download url seems to follow the pattern $GITHUB_REPOSITORY/releases/download/{TAG}/{ASSET_NAME} + # alternatively we can query the asset-id, and browser_download_url, but it seems like an overhead + # asset_id=$(gh api repos/$GITHUB_REPOSITORY/releases/tags/latest | jq -r '.assets[]' | jq --arg DB $DB -c 'select(.name=="release-pod-\( $DB ).zip") | .id) + # download_url=$(gh api repos/$GITHUB_REPOSITORY/releases/assets/$asset_id | jq -r ".browser_download_url") + download_url="https://github.com/$GITHUB_REPOSITORY/releases/download/${{ matrix.tag }}/release-pod.zip" + curl -L $download_url --output release-pod.zip + ls -la + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Start LocalStack + env: + DEBUG: 1 + POD_LOAD_CLI_TIMEOUT: 300 + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + DNS_ADDRESS: 0 + run: | + export SQS_QUEUE="sqs-fargate-queue" + export NETWORK_NAME="localstack-shared-net" + docker network create $NETWORK_NAME + pip install localstack awscli-local[ver1] + docker pull localstack/localstack-pro:${{ matrix.tag }} + # Start LocalStack in the background + LAMBDA_DOCKER_NETWORK=$NETWORK_NAME DOCKER_FLAGS="--network $NETWORK_NAME" DEBUG=1 localstack start -d + # Wait 30 seconds for the LocalStack container to become ready before timing out + echo "Waiting for LocalStack startup..." + localstack wait -t 30 + echo "Startup complete" + + + - name: Inject Pod + run: | + localstack pod load file://release-pod.zip + + # TODO run some tests + - name: Smoke Tests + run: | + # check restored item is still available + count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') + if [ "1" != ${count} ]; then + echo "pod restore failed, expected 1 item in the database" + exit 1 + fi + + SQS_QUEUE="sqs-fargate-queue" + # try to add additional data + awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "my new message"}' + echo "waiting to process the message..." + sleep 3 + count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') + if [ "2" != ${count} ]; then + echo "pod restore failed, expected 2 items in the database" + exit 1 + fi + awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Items[].message.S' + + - name: Show Logs + if: failure() + run: | + localstack logs + + # - name: Send a Slack notification + # if: failure() || github.event_name != 'pull_request' + # uses: ravsamhq/notify-slack-action@v2 + # with: + # status: ${{ job.status }} + # token: ${{ secrets.GITHUB_TOKEN }} + # notification_title: "{workflow} has {status_message}" + # message_format: "{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>" + # footer: "Linked Repo <{repo_url}|{repo}> | <{run_url}|View Workflow run>" + # notify_when: "failure" + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + # - name: Prevent Workflows from getting Stale + # if: always() + # uses: gautamkrishnar/keepalive-workflow@v1 + # with: + # this message should prevent automatic triggering of workflows + # see https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs + # commit_message: "[skip ci] Automated commit by Keepalive Workflow to keep the repository active" From 72a532b1620c46c6143a39a3595e71ca04cd0249 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 19:22:33 +0200 Subject: [PATCH 7/9] fix used localstack-tag for release-workflow; add sleep for tests --- .github/workflows/cloudpod_release.yml | 2 +- .github/workflows/test_cloudpods.yml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml index bc0dcd5..7a0a554 100644 --- a/.github/workflows/cloudpod_release.yml +++ b/.github/workflows/cloudpod_release.yml @@ -47,7 +47,7 @@ jobs: docker network create $NETWORK_NAME pip install localstack awscli-local[ver1] pip install terraform-local - docker pull localstack/localstack-pro:latest + docker pull localstack/localstack-pro:${{ inputs.release-tag || 'latest'}} # Start LocalStack in the background LS_LOG=trace LAMBDA_DOCKER_NETWORK=$NETWORK_NAME DOCKER_FLAGS="--network $NETWORK_NAME" DEBUG=1 localstack start -d # Wait 30 seconds for the LocalStack container to become ready before timing out diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml index 902e94d..1873e7c 100644 --- a/.github/workflows/test_cloudpods.yml +++ b/.github/workflows/test_cloudpods.yml @@ -81,17 +81,18 @@ jobs: - name: Smoke Tests run: | # check restored item is still available + sleep 30 count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') if [ "1" != ${count} ]; then echo "pod restore failed, expected 1 item in the database" exit 1 fi - + SQS_QUEUE="sqs-fargate-queue" # try to add additional data awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "my new message"}' echo "waiting to process the message..." - sleep 3 + sleep 30 count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') if [ "2" != ${count} ]; then echo "pod restore failed, expected 2 items in the database" From 993ea4911475059ea3a7f19c6ce66b5f2c359f32 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Jun 2023 19:41:54 +0200 Subject: [PATCH 8/9] list running docker images for debugging --- .github/workflows/test_cloudpods.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml index 1873e7c..af1330d 100644 --- a/.github/workflows/test_cloudpods.yml +++ b/.github/workflows/test_cloudpods.yml @@ -93,6 +93,8 @@ jobs: awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "my new message"}' echo "waiting to process the message..." sleep 30 + echo "docker ps" + docker ps count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') if [ "2" != ${count} ]; then echo "pod restore failed, expected 2 items in the database" From 9838544911d1b07e786e75a0bfacba67a577fea8 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Fri, 23 Jun 2023 16:30:14 +0200 Subject: [PATCH 9/9] run cloudpod smoke test in loop --- .github/workflows/test_cloudpods.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml index af1330d..a1b1ba0 100644 --- a/.github/workflows/test_cloudpods.yml +++ b/.github/workflows/test_cloudpods.yml @@ -81,9 +81,8 @@ jobs: - name: Smoke Tests run: | # check restored item is still available - sleep 30 - count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') - if [ "1" != ${count} ]; then + actual_count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') + if [ "1" != ${actual_count} ]; then echo "pod restore failed, expected 1 item in the database" exit 1 fi @@ -92,18 +91,26 @@ jobs: # try to add additional data awslocal sqs send-message --queue $SQS_QUEUE --message-body '{"message": "my new message"}' echo "waiting to process the message..." - sleep 30 + + # for debugging echo "docker ps" docker ps - count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') - if [ "2" != ${count} ]; then - echo "pod restore failed, expected 2 items in the database" - exit 1 - fi + + i=0 + actual_count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') + while [ "2" != ${actual_count} ]; do + i=$(( $i+1 )) + if [ 10 -eq ${i} ]; then + echo "pod restore failed, expected 2 items in the database" + exit 1 + fi + sleep 3 + actual_count=$(awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Count') + done awslocal dynamodb scan --table-name sqs-fargate-ddb-table | jq -r '.Items[].message.S' - name: Show Logs - if: failure() + if: always() run: | localstack logs