From 0eae0163dd1e965f657bc33f05949c6a97f8b16e Mon Sep 17 00:00:00 2001 From: Christopher Butler Date: Mon, 10 Nov 2025 10:32:30 -0500 Subject: [PATCH] Build/Deploy Automation Why these changes are being introduced: The ECR Repository infrastruture is in place in dev, stage, and prod AWS accounts, so we have the outputs for the Makefile and the three GHA workflows in TfCloud. Once deployed, these workflows will build/ deploy the container using our usual GitHub-flow model (new PR will push a container to dev, merged PR will push a container to stage, and a tagged release on main will push a container to prod). How this addresses that need: * Add Terraform-generated dev build and dev deploy targets to the Makefile * Create the dev, stage, and prod build/deploy GHA workflows using the text generated by the mitlib-tf-workloads-ecr repository None. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/USE-169 --- .github/workflows/dev-build.yml | 60 +++++++++++++++++++++++++++++++ .github/workflows/prod-deploy.yml | 56 +++++++++++++++++++++++++++++ .github/workflows/stage-build.yml | 59 ++++++++++++++++++++++++++++++ .gitignore | 1 + Makefile | 54 +++++++++++++++++++++++++++- 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dev-build.yml create mode 100644 .github/workflows/prod-deploy.yml create mode 100644 .github/workflows/stage-build.yml diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml new file mode 100644 index 0000000..d9468a3 --- /dev/null +++ b/.github/workflows/dev-build.yml @@ -0,0 +1,60 @@ +### This is the Terraform-generated dev-build.yml workflow for the ### +### timdex-embeddings-dev app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Dev Container Build and Deploy +on: + workflow_dispatch: + pull_request: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Dev Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-dev.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-embeddings-gha-dev" + ECR: "timdex-embeddings-dev" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + # FUNCTION: "" + # PREBUILD: diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml new file mode 100644 index 0000000..276c882 --- /dev/null +++ b/.github/workflows/prod-deploy.yml @@ -0,0 +1,56 @@ +### This is the Terraform-generated prod-promote.yml workflow for the ### +### timdex-embeddings-prod repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. ### + +name: Prod Container Promote +on: + workflow_dispatch: + release: + types: [published] + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Promote + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-promote-prod.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE_STAGE: timdex-embeddings-gha-stage + GHA_ROLE_PROD: timdex-embeddings-gha-prod + ECR_STAGE: "timdex-embeddings-stage" + ECR_PROD: "timdex-embeddings-prod" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + \ No newline at end of file diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml new file mode 100644 index 0000000..d1cf541 --- /dev/null +++ b/.github/workflows/stage-build.yml @@ -0,0 +1,59 @@ +### This is the Terraform-generated stage-build.yml workflow for the ### +### timdex-embeddings-stage app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Stage Container Build and Deploy +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Stage Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-stage.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-embeddings-gha-stage" + ECR: "timdex-embeddings-stage" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + # PREBUILD: diff --git a/.gitignore b/.gitignore index e2261d0..6e23346 100644 --- a/.gitignore +++ b/.gitignore @@ -155,5 +155,6 @@ cython_debug/ .DS_Store output/ .vscode/ +.arch_tag CLAUDE.md \ No newline at end of file diff --git a/Makefile b/Makefile index b999f0a..bee68d9 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,20 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") +### This is the Terraform-generated header for timdex-embeddings-dev. If ### +### this is a Lambda repo, uncomment the FUNCTION line below ### +### and review the other commented lines in the document. ### +ECR_NAME_DEV := timdex-embeddings-dev +ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-embeddings-dev +CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") +### End of Terraform-generated header ### + help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ /^[-_[:alpha:]]+:.?*#/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST) # ensure OS binaries aren't called if naming conflict with Make recipes -.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply +.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply check-arch dist-dev publish-dev docker-clean ############################################## # Python Environment and Dependency commands @@ -85,3 +93,47 @@ docker-shell: # Shell into local container for testing docker-run: # Run main entrypoint + command without arguments docker run timdex-embeddings:latest + + +### Terraform-generated Developer Deploy Commands for Dev environment ### +check-arch: + @ARCH_FILE=".aws-architecture"; \ + if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \ + echo "Invalid CPU_ARCH: $(CPU_ARCH)"; exit 1; \ + fi; \ + if [[ -f $$ARCH_FILE ]]; then \ + echo "latest-$(shell echo $(CPU_ARCH) | cut -d'/' -f2)" > .arch_tag; \ + else \ + echo "latest" > .arch_tag; \ + fi + +dist-dev: check-arch ## Build docker container (intended for developer-based manual build) + @ARCH_TAG=$$(cat .arch_tag); \ + docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \ + docker buildx use $(ECR_NAME_DEV); \ + docker buildx build --platform $(CPU_ARCH) \ + --load \ + --tag $(ECR_URL_DEV):$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$(shell git describe --always) \ + --tag $(ECR_NAME_DEV):$$ARCH_TAG \ + . + +publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish) + @ARCH_TAG=$$(cat .arch_tag); \ + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \ + docker push $(ECR_URL_DEV):$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$(shell git describe --always); \ + echo "Cleaning up dangling Docker images..."; \ + docker image prune -f --filter "dangling=true" + +docker-clean: ## Clean up Docker detritus + @ARCH_TAG=$$(cat .arch_tag); \ + echo "Cleaning up Docker leftovers (containers, images, builders)"; \ + docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$(shell git describe --always) || true; \ + docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \ + docker buildx rm $(ECR_NAME_DEV) || true + @rm -rf .arch_tag