diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml new file mode 100644 index 0000000..d9468a3 --- /dev/null +++ b/.github/workflows/dev-build.yml @@ -0,0 +1,60 @@ +### This is the Terraform-generated dev-build.yml workflow for the ### +### timdex-embeddings-dev app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Dev Container Build and Deploy +on: + workflow_dispatch: + pull_request: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Dev Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-dev.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-embeddings-gha-dev" + ECR: "timdex-embeddings-dev" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + # FUNCTION: "" + # PREBUILD: diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml new file mode 100644 index 0000000..276c882 --- /dev/null +++ b/.github/workflows/prod-deploy.yml @@ -0,0 +1,56 @@ +### This is the Terraform-generated prod-promote.yml workflow for the ### +### timdex-embeddings-prod repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. ### + +name: Prod Container Promote +on: + workflow_dispatch: + release: + types: [published] + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Promote + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-promote-prod.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE_STAGE: timdex-embeddings-gha-stage + GHA_ROLE_PROD: timdex-embeddings-gha-prod + ECR_STAGE: "timdex-embeddings-stage" + ECR_PROD: "timdex-embeddings-prod" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + \ No newline at end of file diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml new file mode 100644 index 0000000..d1cf541 --- /dev/null +++ b/.github/workflows/stage-build.yml @@ -0,0 +1,59 @@ +### This is the Terraform-generated stage-build.yml workflow for the ### +### timdex-embeddings-stage app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Stage Container Build and Deploy +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Stage Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-stage.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-embeddings-gha-stage" + ECR: "timdex-embeddings-stage" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + # PREBUILD: diff --git a/.gitignore b/.gitignore index e2261d0..6e23346 100644 --- a/.gitignore +++ b/.gitignore @@ -155,5 +155,6 @@ cython_debug/ .DS_Store output/ .vscode/ +.arch_tag CLAUDE.md \ No newline at end of file diff --git a/Makefile b/Makefile index b999f0a..bee68d9 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,20 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") +### This is the Terraform-generated header for timdex-embeddings-dev. If ### +### this is a Lambda repo, uncomment the FUNCTION line below ### +### and review the other commented lines in the document. ### +ECR_NAME_DEV := timdex-embeddings-dev +ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-embeddings-dev +CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") +### End of Terraform-generated header ### + help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ /^[-_[:alpha:]]+:.?*#/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST) # ensure OS binaries aren't called if naming conflict with Make recipes -.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply +.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply check-arch dist-dev publish-dev docker-clean ############################################## # Python Environment and Dependency commands @@ -85,3 +93,47 @@ docker-shell: # Shell into local container for testing docker-run: # Run main entrypoint + command without arguments docker run timdex-embeddings:latest + + +### Terraform-generated Developer Deploy Commands for Dev environment ### +check-arch: + @ARCH_FILE=".aws-architecture"; \ + if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \ + echo "Invalid CPU_ARCH: $(CPU_ARCH)"; exit 1; \ + fi; \ + if [[ -f $$ARCH_FILE ]]; then \ + echo "latest-$(shell echo $(CPU_ARCH) | cut -d'/' -f2)" > .arch_tag; \ + else \ + echo "latest" > .arch_tag; \ + fi + +dist-dev: check-arch ## Build docker container (intended for developer-based manual build) + @ARCH_TAG=$$(cat .arch_tag); \ + docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \ + docker buildx use $(ECR_NAME_DEV); \ + docker buildx build --platform $(CPU_ARCH) \ + --load \ + --tag $(ECR_URL_DEV):$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$(shell git describe --always) \ + --tag $(ECR_NAME_DEV):$$ARCH_TAG \ + . + +publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish) + @ARCH_TAG=$$(cat .arch_tag); \ + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \ + docker push $(ECR_URL_DEV):$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$(shell git describe --always); \ + echo "Cleaning up dangling Docker images..."; \ + docker image prune -f --filter "dangling=true" + +docker-clean: ## Clean up Docker detritus + @ARCH_TAG=$$(cat .arch_tag); \ + echo "Cleaning up Docker leftovers (containers, images, builders)"; \ + docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$(shell git describe --always) || true; \ + docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \ + docker buildx rm $(ECR_NAME_DEV) || true + @rm -rf .arch_tag