Skip to content

Commit 8cb1697

Browse files
sthulbleandrodamascena
sthulb
and
leandrodamascena
authored
feat(ci): Deploy to AWS China partitions (#6867)
* feat(ci): Deploy to AWS partitions * fix typo --------- Co-authored-by: Leandro Damascena <lcdama@amazon.pt>
1 parent 9975c14 commit 8cb1697

File tree

2 files changed

+349
-0
lines changed

2 files changed

+349
-0
lines changed

‎.github/workflows/layers_partition_verify.yml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
# Parition Layer Verification
2+
# ---
3+
# This workflow queries the Parition layer info in production only
4+
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
environment:
9+
description: Deployment environment
10+
type: choice
11+
options:
12+
- Gamma
13+
- Prod
14+
required: true
15+
version:
16+
description: Layer version to verify
17+
type: string
18+
required: true
19+
partition_version:
20+
description: Layer version to verify, this is mostly used in Gamma where a version mismatch might exist
21+
type: string
22+
required: false
23+
partition:
24+
description: Partition to deploy to
25+
type: choice
26+
options:
27+
- China
28+
- GovCloud
29+
workflow_call:
30+
inputs:
31+
environment:
32+
description: Deployment environment
33+
type: string
34+
required: true
35+
version:
36+
description: Layer version to verify
37+
type: string
38+
required: true
39+
partition_version:
40+
description: Partition Layer version to verify, this is mostly used in Gamma where a version mismatch might exist
41+
type: string
42+
required: false
43+
44+
name: Layer Verification (Partition)
45+
run-name: Layer Verification (${{ inputs.partition }}) - ${{ inputs.environment }} / Version - ${{ inputs.version }}
46+
47+
permissions: {}
48+
49+
jobs:
50+
setup:
51+
runs-on: ubuntu-latest
52+
outputs:
53+
regions: ${{ format('{0}{1}', steps.regions_china.outputs.regions, steps.regions_govcloud.outputs.regions) }}
54+
parition: ${{ format('{0}{1}', steps.regions_china.outputs.partition, steps.regions_govcloud.outputs.parition) }}
55+
aud: ${{ format('{0}{1}', steps.regions_china.outputs.aud, steps.regions_govcloud.outputs.aud) }}
56+
steps:
57+
- id: regions_china
58+
name: Parition (China)
59+
if: ${{ inputs.partition == 'China' }}
60+
run: |
61+
echo regions='["cn-north-1"]'>> "$GITHUB_OUTPUT"
62+
echo partition='aws-cn'>> "$GITHUB_OUTPUT"
63+
echo aud='sts.amazonaws.com.cn'>> "$GITHUB_OUTPUT"
64+
- id: regions_govcloud
65+
name: Partition (GovCloud)
66+
if: ${{ inputs.partition == 'GovCloud' }}
67+
run: |
68+
echo regions='["us-gov-east-1", "us-gov-west-1"]'>> "$GITHUB_OUTPUT"
69+
echo partition='aws-us-gov'>> "$GITHUB_OUTPUT"
70+
echo aud='sts.amazonaws.com'>> "$GITHUB_OUTPUT"
71+
commercial:
72+
runs-on: ubuntu-latest
73+
permissions:
74+
id-token: write
75+
contents: read
76+
environment: Prod (Readonly)
77+
strategy:
78+
matrix:
79+
layer:
80+
- AWSLambdaPowertoolsPythonV3-python39
81+
- AWSLambdaPowertoolsPythonV3-python310
82+
- AWSLambdaPowertoolsPythonV3-python311
83+
- AWSLambdaPowertoolsPythonV3-python312
84+
- AWSLambdaPowertoolsPythonV3-python313
85+
arch:
86+
- arm64
87+
- x86_64
88+
steps:
89+
- name: Configure AWS Credentials
90+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
91+
with:
92+
role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
93+
aws-region: us-east-1
94+
mask-aws-account-id: true
95+
- name: Output ${{ matrix.layer }}-${{ matrix.arch }}
96+
# fetch the specific layer version information from the us-east-1 commercial region
97+
run: |
98+
aws --region us-east-1 lambda get-layer-version-by-arn --arn 'arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }}' | jq -r '{"Layer Version Arn": .LayerVersionArn, "Version": .Version, "Description": .Description, "Compatible Runtimes": .CompatibleRuntimes[0], "Compatible Architectures": .CompatibleArchitectures[0], "SHA": .Content.CodeSha256} | keys[] as $k | [$k, .[$k]] | @tsv' | column -t -s $'\t'
99+
- name: Store Metadata
100+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
101+
with:
102+
name: ${{ matrix.layer }}-${{ matrix.arch }}.json
103+
path: ${{ matrix.layer }}-${{ matrix.arch }}.json
104+
retention-days: 1
105+
if-no-files-found: error
106+
107+
verify:
108+
name: Verify
109+
needs:
110+
- setup
111+
- commercial
112+
runs-on: ubuntu-latest
113+
permissions:
114+
id-token: write
115+
contents: read
116+
# Environment should interperlate as "GovCloud Prod" or "China Beta"
117+
environment: ${{ inputs.partition }} ${{ inputs.environment }}
118+
strategy:
119+
matrix:
120+
region: ${{ fromJson(needs.setup.outputs.regions) }}
121+
layer:
122+
- AWSLambdaPowertoolsPythonV3-python39
123+
- AWSLambdaPowertoolsPythonV3-python310
124+
- AWSLambdaPowertoolsPythonV3-python311
125+
- AWSLambdaPowertoolsPythonV3-python312
126+
- AWSLambdaPowertoolsPythonV3-python313
127+
arch:
128+
- arm64
129+
- x86_64
130+
steps:
131+
- name: Download Metadata
132+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
133+
with:
134+
name: ${{ matrix.layer }}-${{ matrix.arch }}.json
135+
- id: transform
136+
run: |
137+
echo 'CONVERTED_REGION=${{ matrix.region }}' | tr 'a-z\-' 'A-Z_' >> "$GITHUB_OUTPUT"
138+
- name: Configure AWS Credentials
139+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
140+
with:
141+
role-to-assume: ${{ secrets[format('IAM_ROLE_{0}', steps.transform.outputs.CONVERTED_REGION)] }}
142+
aws-region: ${{ matrix.region}}
143+
mask-aws-account-id: true
144+
audience: ${{ needs.setup.outputs.aud }}
145+
- id: partition_version
146+
name: Partition Layer Version
147+
run: |
148+
echo 'partition_version=$([[ -n "${{ inputs.partition_version}}" ]] && echo ${{ inputs.partition_version}} || echo ${{ inputs.version }} )' >> "$GITHUB_OUTPUT"
149+
- name: Verify Layer
150+
run: |
151+
export layer_output='${{ matrix.layer }}-${{ matrix.arch }}-${{matrix.region}}.json'
152+
aws --region ${{ matrix.region}} lambda get-layer-version-by-arn --arn "arn:${{ needs.setup.outputs.parition }}:lambda:${{ matrix.region}}:${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ steps.partition_version.outputs.partition_version }}" > $layer_output
153+
REMOTE_SHA=$(jq -r '.Content.CodeSha256' $layer_output)
154+
LOCAL_SHA=$(jq -r '.Content.CodeSha256' ${{ matrix.layer }}-${{ matrix.arch }}.json)
155+
test "$REMOTE_SHA" == "$LOCAL_SHA" && echo "SHA OK: ${LOCAL_SHA}" || exit 1
156+
jq -s -r '["Layer Arn", "Runtimes", "Version", "Description", "SHA256"], ([.[0], .[1]] | .[] | [.LayerArn, (.CompatibleRuntimes | join("/")), .Version, .Description, .Content.CodeSha256]) |@tsv' ${{ matrix.layer }}-${{ matrix.arch }}.json $layer_output | column -t -s $'\t'

‎.github/workflows/layers_partitions.yml‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
# Partitioned Layer Publish
2+
# ---
3+
# This workflow publishes a specific layer version in an AWS account based on the environment input.
4+
#
5+
# We pull each the version of the layer and store them as artifacts, the we upload them to each of the Partitioned AWS accounts.
6+
#
7+
# A number of safety checks are performed to ensure safety.
8+
9+
on:
10+
workflow_dispatch:
11+
inputs:
12+
environment:
13+
description: Deployment environment
14+
type: choice
15+
options:
16+
- Gamma
17+
- Prod
18+
required: true
19+
version:
20+
description: Layer version to duplicate
21+
type: string
22+
required: true
23+
partition:
24+
description: Partition to deploy to
25+
type: choice
26+
options:
27+
- China
28+
- GovCloud
29+
workflow_call:
30+
inputs:
31+
environment:
32+
description: Deployment environment
33+
type: string
34+
required: true
35+
version:
36+
description: Layer version to duplicate
37+
type: string
38+
required: true
39+
40+
name: Layer Deployment (Partitions)
41+
run-name: Layer Deployment (${{ inputs.partition }}) - ${{ inputs.environment }} / Version - ${{ inputs.version }}
42+
43+
permissions:
44+
contents: read
45+
46+
jobs:
47+
setup:
48+
runs-on: ubuntu-latest
49+
outputs:
50+
regions: ${{ format('{0}{1}', steps.regions_china.outputs.regions, steps.regions_govcloud.outputs.regions) }}
51+
partition: ${{ format('{0}{1}', steps.regions_china.outputs.partition, steps.regions_govcloud.outputs.partition) }}
52+
aud: ${{ format('{0}{1}', steps.regions_china.outputs.aud, steps.regions_govcloud.outputs.aud) }}
53+
steps:
54+
- id: regions_china
55+
name: Partition (China)
56+
if: ${{ inputs.partition == 'China' }}
57+
run: |
58+
echo regions='["cn-north-1"]'>> "$GITHUB_OUTPUT"
59+
echo partition='aws-cn'>> "$GITHUB_OUTPUT"
60+
echo aud='sts.amazonaws.com.cn'>> "$GITHUB_OUTPUT"
61+
- id: regions_govcloud
62+
name: Partition (GovCloud)
63+
if: ${{ inputs.partition == 'GovCloud' }}
64+
run: |
65+
echo regions='["us-gov-east-1", "us-gov-west-1"]'>> "$GITHUB_OUTPUT"
66+
echo partition='aws-us-gov'>> "$GITHUB_OUTPUT"
67+
echo aud='sts.amazonaws.com'>> "$GITHUB_OUTPUT"
68+
download:
69+
runs-on: ubuntu-latest
70+
permissions:
71+
id-token: write
72+
contents: read
73+
environment: Prod (Readonly)
74+
strategy:
75+
matrix:
76+
layer:
77+
- AWSLambdaPowertoolsPythonV3-python39
78+
- AWSLambdaPowertoolsPythonV3-python310
79+
- AWSLambdaPowertoolsPythonV3-python311
80+
- AWSLambdaPowertoolsPythonV3-python312
81+
- AWSLambdaPowertoolsPythonV3-python313
82+
arch:
83+
- arm64
84+
- x86_64
85+
steps:
86+
- name: Configure AWS Credentials
87+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
88+
with:
89+
role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
90+
aws-region: us-east-1
91+
mask-aws-account-id: true
92+
- name: Grab Zip
93+
run: |
94+
aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} --query 'Content.Location' | xargs curl -L -o ${{ matrix.layer }}_${{ matrix.arch }}.zip
95+
aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} > ${{ matrix.layer }}_${{ matrix.arch }}.json
96+
- name: Store Zip
97+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
98+
with:
99+
name: ${{ matrix.layer }}_${{ matrix.arch }}.zip
100+
path: ${{ matrix.layer }}_${{ matrix.arch }}.zip
101+
retention-days: 1
102+
if-no-files-found: error
103+
- name: Store Metadata
104+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
105+
with:
106+
name: ${{ matrix.layer }}_${{ matrix.arch }}.json
107+
path: ${{ matrix.layer }}_${{ matrix.arch }}.json
108+
retention-days: 1
109+
if-no-files-found: error
110+
111+
copy:
112+
name: Copy
113+
needs:
114+
- setup
115+
- download
116+
runs-on: ubuntu-latest
117+
permissions:
118+
id-token: write
119+
contents: read
120+
# Environment should interperlate as "GovCloud Prod" or "China Beta"
121+
environment: ${{ inputs.partition }} ${{ inputs.environment }}
122+
strategy:
123+
matrix:
124+
region: ${{ fromJson(needs.setup.outputs.regions) }}
125+
layer:
126+
- AWSLambdaPowertoolsPythonV3-python39
127+
- AWSLambdaPowertoolsPythonV3-python310
128+
- AWSLambdaPowertoolsPythonV3-python311
129+
- AWSLambdaPowertoolsPythonV3-python312
130+
- AWSLambdaPowertoolsPythonV3-python313
131+
arch:
132+
- arm64
133+
- x86_64
134+
steps:
135+
- name: Download Zip
136+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
137+
with:
138+
name: ${{ matrix.layer }}_${{ matrix.arch }}.zip
139+
- name: Download Metadata
140+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
141+
with:
142+
name: ${{ matrix.layer }}_${{ matrix.arch }}.json
143+
- name: Verify Layer Signature
144+
run: |
145+
SHA=$(jq -r '.Content.CodeSha256' '${{ matrix.layer }}_${{ matrix.arch }}.json')
146+
test "$(openssl dgst -sha256 -binary ${{ matrix.layer }}_${{ matrix.arch }}.zip | openssl enc -base64)" == "$SHA" && echo "SHA OK: ${SHA}" || exit 1
147+
- id: transform
148+
run: |
149+
echo 'CONVERTED_REGION=${{ matrix.region }}' | tr 'a-z\-' 'A-Z_' >> "$GITHUB_OUTPUT"
150+
- name: Configure AWS Credentials
151+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
152+
with:
153+
role-to-assume: ${{ secrets[format('IAM_ROLE_{0}', steps.transform.outputs.CONVERTED_REGION)] }}
154+
aws-region: ${{ matrix.region}}
155+
mask-aws-account-id: true
156+
audience: ${{ needs.setup.outputs.aud }}
157+
- name: Create Layer
158+
id: create-layer
159+
run: |
160+
cat '${{ matrix.layer }}-${{ matrix.arch }}.json' | jq '{"LayerName": "${{ matrix.layer }}-${{ matrix.arch }}", "Description": .Description, "CompatibleRuntimes": .CompatibleRuntimes, "CompatibleArchitectures": .CompatibleArchitectures, "LicenseInfo": .LicenseInfo}' > input.json
161+
162+
LAYER_VERSION=$(aws --region ${{ matrix.region}} lambda publish-layer-version \
163+
--zip-file 'fileb://./${{ matrix.layer }}-${{ matrix.arch }}.zip' \
164+
--cli-input-json file://./input.json \
165+
--query 'Version' \
166+
--output text)
167+
168+
echo "LAYER_VERSION=$LAYER_VERSION" >> "$GITHUB_OUTPUT"
169+
170+
aws --region ${{ matrix.region}} lambda add-layer-version-permission \
171+
--layer-name ${{ matrix.layer }}-${{ matrix.arch }} \
172+
--statement-id 'PublicLayer' \
173+
--action lambda:GetLayerVersion \
174+
--principal '*' \
175+
--version-number "$LAYER_VERSION"
176+
- name: Verify Layer
177+
env:
178+
LAYER_VERSION: ${{ steps.create-layer.outputs.LAYER_VERSION }}
179+
run: |
180+
export layer_output='${{ matrix.layer }}-${{ matrix.arch }}-${{matrix.region}}.json'
181+
aws --region ${{ matrix.region}} lambda get-layer-version-by-arn --arn 'arn:${{ needs.setup.outputs.partition }}:lambda:${{ matrix.region}}:${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ env.LAYER_VERSION }}' > $layer_output
182+
REMOTE_SHA=$(jq -r '.Content.CodeSha256' $layer_output)
183+
LOCAL_SHA=$(jq -r '.Content.CodeSha256' '${{ matrix.layer }}-${{ matrix.arch }}.json')
184+
test "$REMOTE_SHA" == "$LOCAL_SHA" && echo "SHA OK: ${LOCAL_SHA}" || exit 1
185+
jq -s -r '["Layer Arn", "Runtimes", "Version", "Description", "SHA256"], ([.[0], .[1]] | .[] | [.LayerArn, (.CompatibleRuntimes | join("/")), .Version, .Description, .Content.CodeSha256]) |@tsv' '${{ matrix.layer }}-${{ matrix.arch }}.json' $layer_output | column -t -s $'\t'
186+
187+
- name: Store Metadata - ${{ matrix.region }}
188+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
189+
with:
190+
name: ${{ matrix.layer }}-${{ matrix.arch }}-${{ matrix.region }}.json
191+
path: ${{ matrix.layer }}-${{ matrix.arch }}-${{ matrix.region }}.json
192+
retention-days: 1
193+
if-no-files-found: error

0 commit comments

Comments
 (0)