Skip to content

Commit 66d2feb

Browse files
brightsparcbrightsparc
committed
Adding batch pipeline
1 parent 15b86f8 commit 66d2feb

File tree

2 files changed

+256
-1
lines changed

2 files changed

+256
-1
lines changed

‎.github/workflows/build-batch-pipeline.yml‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
# Create a unified Build and Deploy pipeline that uses approval gates
2+
# Build Model –> Deploy to staging with approval –> Deploy to prod with approval.
3+
# https://timheuer.com/blog/add-approval-workflow-to-github-actions/
4+
5+
name: Build and Batch Pipeline
6+
7+
on: [push, pull_request] # Optionally filter on branch
8+
9+
jobs:
10+
build:
11+
name: Build Model
12+
runs-on: ubuntu-latest
13+
environment:
14+
name: development
15+
defaults:
16+
run:
17+
shell: bash
18+
working-directory: ./build_pipeline
19+
steps:
20+
- name: Checkout
21+
uses: actions/checkout@v2
22+
23+
- name: Config Environment
24+
id: env-name
25+
env:
26+
PROJECT_NAME: ${{ github.event.repository.name }}
27+
run: |
28+
echo "Project name: $PROJECT_NAME"
29+
echo "::set-output name=project_name::$PROJECT_NAME"
30+
31+
- name: Setup Python
32+
uses: actions/setup-python@v2
33+
with:
34+
python-version: "3.8"
35+
architecture: "x64"
36+
37+
- name: Setup Node
38+
uses: actions/setup-node@v2
39+
with:
40+
node-version: "12"
41+
architecture: "x64"
42+
cache: npm
43+
44+
- name: Install Requirements
45+
run: |
46+
npm install -g aws-cdk # Install cdk
47+
pip install --requirement requirements.txt
48+
49+
- name: Configure AWS Credentials
50+
id: creds
51+
uses: aws-actions/configure-aws-credentials@v1
52+
with:
53+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
54+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
55+
aws-region: ${{ secrets.AWS_REGION }}
56+
role-to-assume: ${{ secrets.AWS_SAGEMAKER_ROLE }}
57+
role-duration-seconds: 1200
58+
59+
- name: Build Pipeline
60+
id: build-pipeline
61+
env:
62+
SAGEMAKER_PROJECT_NAME: ${{ steps.env-name.outputs.project_name }}
63+
SAGEMAKER_PIPELINE_NAME: ${{ steps.env-name.outputs.project_name }}-pipeline
64+
SAGEMAKER_PIPELINE_DESCRIPTION: "SageMaker pipeline created from GitHub actions"
65+
SAGEMAKER_PIPELINE_ROLE_ARN: ${{ secrets.AWS_SAGEMAKER_ROLE }}
66+
AWS_REGION: ${{ secrets.AWS_REGION }}
67+
run: |
68+
export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text`
69+
echo "Project id: $SAGEMAKER_PROJECT_ID"
70+
export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION
71+
echo "Artifact Bucket: $ARTIFACT_BUCKET"
72+
npx cdk synth --path-metadata false --asset-metadata=false > drift-pipeline.yml
73+
echo "::set-output name=pipeline_name::$SAGEMAKER_PIPELINE_NAME"
74+
75+
- name: Print template
76+
run: cat drift-pipeline.yml
77+
78+
- name: Create CFN Pipeline
79+
id: deploy-pipeline
80+
uses: aws-actions/aws-cloudformation-github-deploy@v1
81+
with:
82+
name: sagemaker-${{ steps.build-pipeline.outputs.pipeline_name }}
83+
template: ./build_pipeline/drift-pipeline.yml # Need to specify working-directory
84+
no-fail-on-empty-changeset: "1"
85+
86+
- name: Start Pipeline
87+
id: start-pipeline # TODO: Run python code that waits for pipeline to complete
88+
run: aws sagemaker start-pipeline-execution --pipeline-name ${{ steps.build-pipeline.outputs.pipeline_name }} --pipeline-parameters Name=InputSource,Value=GitHubAction#${{ github.run_number }}
89+
90+
- name: Upload template
91+
uses: actions/upload-artifact@v2
92+
with:
93+
name: drift-pipeline
94+
path: ./build_pipeline/drift-pipeline.yml
95+
96+
deploy_staging:
97+
needs: build
98+
name: Deploy to staging
99+
runs-on: ubuntu-latest
100+
environment:
101+
name: staging # Use different environment that optionally requires approval
102+
defaults:
103+
run:
104+
shell: bash
105+
working-directory: ./batch_pipeline
106+
steps:
107+
- name: Checkout
108+
uses: actions/checkout@v2
109+
110+
- name: Config Environment
111+
id: env-name
112+
env:
113+
PROJECT_NAME: ${{ github.event.repository.name }}
114+
run: |
115+
echo "Project name: $PROJECT_NAME"
116+
echo "::set-output name=project_name::$PROJECT_NAME"
117+
118+
- name: Setup Python
119+
uses: actions/setup-python@v2
120+
with:
121+
python-version: "3.8"
122+
architecture: "x64"
123+
124+
- name: Setup Node
125+
uses: actions/setup-node@v2
126+
with:
127+
node-version: "12"
128+
architecture: "x64"
129+
cache: npm
130+
131+
- name: Install Requirements
132+
run: |
133+
npm install -g aws-cdk # Install cdk
134+
pip install --requirement requirements.txt
135+
136+
- name: Configure AWS Credentials
137+
id: creds
138+
uses: aws-actions/configure-aws-credentials@v1
139+
with:
140+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
141+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
142+
aws-region: ${{ secrets.AWS_REGION }}
143+
role-to-assume: ${{ secrets.AWS_SAGEMAKER_ROLE }}
144+
role-duration-seconds: 1200
145+
146+
- name: Build Templates
147+
id: build-templates
148+
env:
149+
SAGEMAKER_PROJECT_NAME: ${{ steps.env-name.outputs.project_name }}
150+
SAGEMAKER_EXECUTION_ROLE_ARN: ${{ secrets.AWS_SAGEMAKER_ROLE }}
151+
AWS_REGION: ${{ secrets.AWS_REGION }}
152+
run: |
153+
export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text`
154+
echo "Project id: $SAGEMAKER_PROJECT_ID"
155+
export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION
156+
echo "Artifact Bucket: $ARTIFACT_BUCKET"
157+
npx cdk synth drift-batch-staging --path-metadata false --asset-metadata=false > drift-batch-staging.yml
158+
159+
- name: Print template
160+
run: cat drift-batch-staging.yml
161+
162+
- name: Deploy Staging
163+
id: deploy-pipeline
164+
uses: aws-actions/aws-cloudformation-github-deploy@v1
165+
with:
166+
name: sagemaker-${{ steps.env-name.outputs.project_name }}-deploy-staging
167+
template: ./batch_pipeline/drift-batch-staging.yml # Need to specify working-directory
168+
no-fail-on-empty-changeset: "1"
169+
170+
- name: Upload template
171+
uses: actions/upload-artifact@v2
172+
with:
173+
name: drift-batch-staging
174+
path: ./batch_pipeline/drift-batch-staging.yml
175+
176+
deploy_prod:
177+
needs: deploy_staging
178+
name: Deploy to prod
179+
if: ${{ github.ref == 'refs/heads/main' }} # Filter to only run on main branch
180+
runs-on: ubuntu-latest
181+
environment:
182+
name: prod # Use different environment that requires approval
183+
defaults:
184+
run:
185+
shell: bash
186+
working-directory: ./batch_pipeline
187+
steps:
188+
- name: Checkout
189+
uses: actions/checkout@v2
190+
191+
- name: Config Environment
192+
id: env-name
193+
env:
194+
PROJECT_NAME: ${{ github.event.repository.name }}
195+
run: |
196+
echo "Project name: $PROJECT_NAME"
197+
echo "::set-output name=project_name::$PROJECT_NAME"
198+
199+
- name: Setup Python
200+
uses: actions/setup-python@v2
201+
with:
202+
python-version: "3.8"
203+
architecture: "x64"
204+
205+
- name: Setup Node
206+
uses: actions/setup-node@v2
207+
with:
208+
node-version: "12"
209+
architecture: "x64"
210+
cache: npm
211+
212+
- name: Install Requirements
213+
run: |
214+
npm install -g aws-cdk # Install cdk
215+
pip install --requirement requirements.txt
216+
217+
- name: Configure AWS Credentials
218+
id: creds
219+
uses: aws-actions/configure-aws-credentials@v1
220+
with:
221+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
222+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
223+
aws-region: ${{ secrets.AWS_REGION }}
224+
role-to-assume: ${{ secrets.AWS_SAGEMAKER_ROLE }}
225+
role-duration-seconds: 1200
226+
227+
- name: Build Templates
228+
id: build-templates
229+
env:
230+
SAGEMAKER_PROJECT_NAME: ${{ steps.env-name.outputs.project_name }}
231+
SAGEMAKER_EXECUTION_ROLE_ARN: ${{ secrets.AWS_SAGEMAKER_ROLE }}
232+
AWS_REGION: ${{ secrets.AWS_REGION }}
233+
run: |
234+
export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text`
235+
echo "Project id: $SAGEMAKER_PROJECT_ID"
236+
export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION
237+
echo "Artifact Bucket: $ARTIFACT_BUCKET"
238+
npx cdk synth drift-batch-prod --path-metadata false --asset-metadata=false > drift-batch-prod.yml
239+
240+
- name: Print Template
241+
run: cat drift-batch-prod.yml
242+
243+
- name: Deploy Prod
244+
id: deploy-pipeline
245+
uses: aws-actions/aws-cloudformation-github-deploy@v1
246+
with:
247+
name: sagemaker-${{ steps.env-name.outputs.project_name }}-deploy-prod
248+
template: ./batch_pipeline/drift-batch-prod.yml # Need to specify working-directory
249+
no-fail-on-empty-changeset: "1"
250+
251+
- name: Upload template
252+
uses: actions/upload-artifact@v2
253+
with:
254+
name: drift-batch-prod
255+
path: ./batch_pipeline/drift-batch-prod.yml

‎.github/workflows/build-deploy-pipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Build Model –> Deploy to staging with approval –> Deploy to prod with approval.
33
# https://timheuer.com/blog/add-approval-workflow-to-github-actions/
44

5-
name: Build and Deploy
5+
name: Build and Deploy Pipeline
66

77
on: [push, pull_request] # Optionally filter on branch
88

0 commit comments

Comments
 (0)