Skip to content

Commit 0eae016

Browse files
cabutlermitcabutlermit
committed
Build/Deploy Automation
Why these changes are being introduced: The ECR Repository infrastruture is in place in dev, stage, and prod AWS accounts, so we have the outputs for the Makefile and the three GHA workflows in TfCloud. Once deployed, these workflows will build/ deploy the container using our usual GitHub-flow model (new PR will push a container to dev, merged PR will push a container to stage, and a tagged release on main will push a container to prod). How this addresses that need: * Add Terraform-generated dev build and dev deploy targets to the Makefile * Create the dev, stage, and prod build/deploy GHA workflows using the text generated by the mitlib-tf-workloads-ecr repository None. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/USE-169
1 parent 0e295af commit 0eae016

File tree

5 files changed

+229
-1
lines changed

5 files changed

+229
-1
lines changed

‎.github/workflows/dev-build.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
### This is the Terraform-generated dev-build.yml workflow for the ###
2+
### timdex-embeddings-dev app repository. ###
3+
### If this is a Lambda repo, uncomment the FUNCTION line at the end of ###
4+
### the document. If the container requires any additional pre-build ###
5+
### commands, uncomment and edit the PREBUILD line at the end of the ###
6+
### document. ###
7+
8+
name: Dev Container Build and Deploy
9+
on:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths-ignore:
15+
- '.github/**'
16+
17+
permissions:
18+
id-token: write
19+
contents: read
20+
21+
jobs:
22+
prep:
23+
name: Prep for Build
24+
runs-on: ubuntu-latest
25+
outputs:
26+
cpuarch: ${{ steps.setarch.outputs.cpuarch }}
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v5
30+
31+
- name: Set CPU Architecture
32+
id: setarch
33+
run: |
34+
echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY
35+
if [[ -f .aws-architecture ]]; then
36+
ARCH=$(cat .aws-architecture)
37+
echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY
38+
else
39+
ARCH="linux/amd64"
40+
echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY
41+
fi
42+
if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then
43+
echo "$ARCH is INVALID architecture!"
44+
echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY
45+
exit 1
46+
fi
47+
echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT
48+
49+
deploy:
50+
needs: prep
51+
name: Dev Deploy
52+
uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-dev.yml@main
53+
secrets: inherit
54+
with:
55+
AWS_REGION: "us-east-1"
56+
GHA_ROLE: "timdex-embeddings-gha-dev"
57+
ECR: "timdex-embeddings-dev"
58+
CPU_ARCH: ${{ needs.prep.outputs.cpuarch }}
59+
# FUNCTION: ""
60+
# PREBUILD:

‎.github/workflows/prod-deploy.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
### This is the Terraform-generated prod-promote.yml workflow for the ###
2+
### timdex-embeddings-prod repository. ###
3+
### If this is a Lambda repo, uncomment the FUNCTION line at the end of ###
4+
### the document. ###
5+
6+
name: Prod Container Promote
7+
on:
8+
workflow_dispatch:
9+
release:
10+
types: [published]
11+
12+
permissions:
13+
id-token: write
14+
contents: read
15+
16+
jobs:
17+
prep:
18+
name: Prep for Promote
19+
runs-on: ubuntu-latest
20+
outputs:
21+
cpuarch: ${{ steps.setarch.outputs.cpuarch }}
22+
steps:
23+
- name: Checkout
24+
uses: actions/checkout@v5
25+
26+
- name: Set CPU Architecture
27+
id: setarch
28+
run: |
29+
echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY
30+
if [[ -f .aws-architecture ]]; then
31+
ARCH=$(cat .aws-architecture)
32+
echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY
33+
else
34+
ARCH="linux/amd64"
35+
echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY
36+
fi
37+
if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then
38+
echo "$ARCH is INVALID architecture!"
39+
echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY
40+
exit 1
41+
fi
42+
echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT
43+
44+
deploy:
45+
needs: prep
46+
name: Deploy
47+
uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-promote-prod.yml@main
48+
secrets: inherit
49+
with:
50+
AWS_REGION: "us-east-1"
51+
GHA_ROLE_STAGE: timdex-embeddings-gha-stage
52+
GHA_ROLE_PROD: timdex-embeddings-gha-prod
53+
ECR_STAGE: "timdex-embeddings-stage"
54+
ECR_PROD: "timdex-embeddings-prod"
55+
CPU_ARCH: ${{ needs.prep.outputs.cpuarch }}
56+

‎.github/workflows/stage-build.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
### This is the Terraform-generated stage-build.yml workflow for the ###
2+
### timdex-embeddings-stage app repository. ###
3+
### If this is a Lambda repo, uncomment the FUNCTION line at the end of ###
4+
### the document. If the container requires any additional pre-build ###
5+
### commands, uncomment and edit the PREBUILD line at the end of the ###
6+
### document. ###
7+
8+
name: Stage Container Build and Deploy
9+
on:
10+
workflow_dispatch:
11+
push:
12+
branches:
13+
- main
14+
paths-ignore:
15+
- '.github/**'
16+
17+
permissions:
18+
id-token: write
19+
contents: read
20+
21+
jobs:
22+
prep:
23+
name: Prep for Build
24+
runs-on: ubuntu-latest
25+
outputs:
26+
cpuarch: ${{ steps.setarch.outputs.cpuarch }}
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v5
30+
31+
- name: Set CPU Architecture
32+
id: setarch
33+
run: |
34+
echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY
35+
if [[ -f .aws-architecture ]]; then
36+
ARCH=$(cat .aws-architecture)
37+
echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY
38+
else
39+
ARCH="linux/amd64"
40+
echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY
41+
fi
42+
if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then
43+
echo "$ARCH is INVALID architecture!"
44+
echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY
45+
exit 1
46+
fi
47+
echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT
48+
49+
deploy:
50+
needs: prep
51+
name: Stage Deploy
52+
uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-stage.yml@main
53+
secrets: inherit
54+
with:
55+
AWS_REGION: "us-east-1"
56+
GHA_ROLE: "timdex-embeddings-gha-stage"
57+
ECR: "timdex-embeddings-stage"
58+
CPU_ARCH: ${{ needs.prep.outputs.cpuarch }}
59+
# PREBUILD:

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,5 +155,6 @@ cython_debug/
155155
.DS_Store
156156
output/
157157
.vscode/
158+
.arch_tag
158159

159160
CLAUDE.md

‎Makefile‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,20 @@ SHELL=/bin/bash
22
DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
33
CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64")
44

5+
### This is the Terraform-generated header for timdex-embeddings-dev. If ###
6+
### this is a Lambda repo, uncomment the FUNCTION line below ###
7+
### and review the other commented lines in the document. ###
8+
ECR_NAME_DEV := timdex-embeddings-dev
9+
ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-embeddings-dev
10+
CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64")
11+
### End of Terraform-generated header ###
12+
513
help: # Preview Makefile commands
614
@awk 'BEGIN { FS = ":.*#"; print "Usage: make <target>\n\nTargets:" } \
715
/^[-_[:alpha:]]+:.?*#/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
816

917
# ensure OS binaries aren't called if naming conflict with Make recipes
10-
.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply
18+
.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply check-arch dist-dev publish-dev docker-clean
1119

1220
##############################################
1321
# Python Environment and Dependency commands
@@ -85,3 +93,47 @@ docker-shell: # Shell into local container for testing
8593

8694
docker-run: # Run main entrypoint + command without arguments
8795
docker run timdex-embeddings:latest
96+
97+
98+
### Terraform-generated Developer Deploy Commands for Dev environment ###
99+
check-arch:
100+
@ARCH_FILE=".aws-architecture"; \
101+
if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \
102+
echo "Invalid CPU_ARCH: $(CPU_ARCH)"; exit 1; \
103+
fi; \
104+
if [[ -f $$ARCH_FILE ]]; then \
105+
echo "latest-$(shell echo $(CPU_ARCH) | cut -d'/' -f2)" > .arch_tag; \
106+
else \
107+
echo "latest" > .arch_tag; \
108+
fi
109+
110+
dist-dev: check-arch ## Build docker container (intended for developer-based manual build)
111+
@ARCH_TAG=$$(cat .arch_tag); \
112+
docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \
113+
docker buildx use $(ECR_NAME_DEV); \
114+
docker buildx build --platform $(CPU_ARCH) \
115+
--load \
116+
--tag $(ECR_URL_DEV):$$ARCH_TAG \
117+
--tag $(ECR_URL_DEV):make-$$ARCH_TAG \
118+
--tag $(ECR_URL_DEV):make-$(shell git describe --always) \
119+
--tag $(ECR_NAME_DEV):$$ARCH_TAG \
120+
.
121+
122+
publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish)
123+
@ARCH_TAG=$$(cat .arch_tag); \
124+
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \
125+
docker push $(ECR_URL_DEV):$$ARCH_TAG; \
126+
docker push $(ECR_URL_DEV):make-$$ARCH_TAG; \
127+
docker push $(ECR_URL_DEV):make-$(shell git describe --always); \
128+
echo "Cleaning up dangling Docker images..."; \
129+
docker image prune -f --filter "dangling=true"
130+
131+
docker-clean: ## Clean up Docker detritus
132+
@ARCH_TAG=$$(cat .arch_tag); \
133+
echo "Cleaning up Docker leftovers (containers, images, builders)"; \
134+
docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \
135+
docker rmi -f $(ECR_URL_DEV):make-$$ARCH_TAG; \
136+
docker rmi -f $(ECR_URL_DEV):make-$(shell git describe --always) || true; \
137+
docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \
138+
docker buildx rm $(ECR_NAME_DEV) || true
139+
@rm -rf .arch_tag

0 commit comments

Comments
 (0)