Skip to content

Commit 6faa95f

Browse files
mmckymmcky
authored
Setup EC2 based builds for preview, publish, cache for GPU support (#280)
1 parent b328338 commit 6faa95f

File tree

3 files changed

+143
-70
lines changed

3 files changed

+143
-70
lines changed

‎.github/workflows/cache.yml‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,35 @@ on:
44
branches:
55
- main
66
jobs:
7-
tests:
7+
deploy-runner:
88
runs-on: ubuntu-latest
99
steps:
10-
- name: Checkout
11-
uses: actions/checkout@v2
10+
- uses: iterative/setup-cml@v1
11+
- uses: actions/checkout@v3
12+
with:
13+
ref: ${{ github.event.pull_request.head.sha }}
14+
- name: Deploy runner on EC2
15+
env:
16+
REPO_TOKEN: ${{ secrets.QUANTECON_SERVICES_PAT }}
17+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
18+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
19+
run: |
20+
cml runner launch \
21+
--cloud=aws \
22+
--cloud-region=us-west-2 \
23+
--cloud-type=p3.2xlarge \
24+
--labels=cml-gpu \
25+
--cloud-hdd-size=40
26+
cache:
27+
needs: deploy-runner
28+
runs-on: [self-hosted, cml-gpu]
29+
container:
30+
image: docker://nvidia/cuda:11.2.1-devel-ubuntu20.04
31+
options: --gpus all
32+
steps:
33+
- uses: actions/checkout@v3
34+
with:
35+
ref: ${{ github.event.pull_request.head.sha }}
1236
- name: Setup Anaconda
1337
uses: conda-incubator/setup-miniconda@v2
1438
with:
@@ -18,32 +42,17 @@ jobs:
1842
python-version: 3.9
1943
environment-file: environment.yml
2044
activate-environment: quantecon
21-
- name: Install latex dependencies
45+
- name: Install Jax and Upgrade CUDA
46+
shell: bash -l {0}
2247
run: |
23-
sudo apt-get -qq update
24-
sudo apt-get install -y \
25-
texlive-latex-recommended \
26-
texlive-latex-extra \
27-
texlive-fonts-recommended \
28-
texlive-fonts-extra \
29-
texlive-xetex \
30-
latexmk \
31-
xindy \
32-
dvipng \
33-
cm-super
48+
pip install --upgrade "jax[cuda]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
49+
nvidia-smi
3450
- name: Build HTML
3551
shell: bash -l {0}
3652
run: |
3753
jb build lectures --path-output ./ -W --keep-going
38-
- name: Upload Execution Reports
39-
uses: actions/upload-artifact@v2
40-
if: failure()
41-
with:
42-
name: execution-reports
43-
path: _build/html/reports
4454
- name: Upload "_build" folder (cache)
45-
uses: actions/upload-artifact@v2
55+
uses: actions/upload-artifact@v3
4656
with:
4757
name: build-cache
48-
path: _build
49-
58+
path: _build

‎.github/workflows/ci.yml‎

Lines changed: 53 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,35 @@
11
name: Build Project [using jupyter-book]
22
on: [pull_request]
33
jobs:
4-
tests:
4+
deploy-runner:
55
runs-on: ubuntu-latest
66
steps:
7-
- name: Checkout
8-
uses: actions/checkout@v2
7+
- uses: iterative/setup-cml@v1
8+
- uses: actions/checkout@v3
9+
with:
10+
ref: ${{ github.event.pull_request.head.sha }}
11+
- name: Deploy runner on EC2
12+
env:
13+
REPO_TOKEN: ${{ secrets.QUANTECON_SERVICES_PAT }}
14+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
15+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
16+
run: |
17+
cml runner launch \
18+
--cloud=aws \
19+
--cloud-region=us-west-2 \
20+
--cloud-type=p3.2xlarge \
21+
--labels=cml-gpu \
22+
--cloud-hdd-size=40
23+
preview:
24+
needs: deploy-runner
25+
runs-on: [self-hosted, cml-gpu]
26+
container:
27+
image: docker://nvidia/cuda:11.2.1-devel-ubuntu20.04
28+
options: --gpus all
29+
steps:
30+
- uses: actions/checkout@v3
31+
with:
32+
ref: ${{ github.event.pull_request.head.sha }}
933
- name: Setup Anaconda
1034
uses: conda-incubator/setup-miniconda@v2
1135
with:
@@ -15,20 +39,25 @@ jobs:
1539
python-version: 3.9
1640
environment-file: environment.yml
1741
activate-environment: quantecon
42+
- name: Install Jax and Upgrade CUDA
43+
shell: bash -l {0}
44+
run: |
45+
pip install --upgrade "jax[cuda]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
46+
nvidia-smi
1847
- name: Install latex dependencies
48+
shell: bash -l {0}
1949
run: |
20-
sudo apt-get -qq update
21-
sudo apt-get install -y \
50+
apt-get -qq update
51+
export DEBIAN_FRONTEND=noninteractive
52+
apt-get install -y tzdata
53+
apt-get install -y \
2254
texlive-latex-recommended \
2355
texlive-latex-extra \
2456
texlive-fonts-recommended \
2557
texlive-fonts-extra \
2658
texlive-xetex \
2759
latexmk \
28-
xindy \
29-
dvipng \
30-
cm-super \
31-
msttcorefonts
60+
xindy
3261
- name: Display Conda Environment Versions
3362
shell: bash -l {0}
3463
run: conda list
@@ -43,33 +72,34 @@ jobs:
4372
name: build-cache
4473
path: _build
4574
# Build Assets (Download Notebooks and PDF via LaTeX)
46-
- name: Build PDF from LaTeX
47-
shell: bash -l {0}
48-
run: |
49-
jb build lectures --builder pdflatex --path-output ./ -n --keep-going
50-
mkdir -p _build/html/_pdf
51-
cp -u _build/latex/*.pdf _build/html/_pdf
5275
- name: Build Download Notebooks (sphinx-tojupyter)
5376
shell: bash -l {0}
5477
run: |
55-
jb build lectures --path-output ./ --builder=custom --custom-builder=jupyter
78+
jb build lectures --path-output ./ --builder=custom --custom-builder=jupyter -n -W --keep-going
5679
mkdir -p _build/html/_notebooks
5780
cp -u _build/jupyter/*.ipynb _build/html/_notebooks
58-
# Build HTML (Website)
81+
- name: Build PDF from LaTeX
82+
shell: bash -l {0}
83+
run: |
84+
jb build lectures --builder pdflatex --path-output ./ -n -W --keep-going
85+
mkdir _build/html/_pdf
86+
cp -u _build/latex/*.pdf _build/html/_pdf
87+
# Final Build of HTML
5988
- name: Build HTML
6089
shell: bash -l {0}
6190
run: |
62-
jb build -W --keep-going lectures --path-output ./
63-
- name: Save Build as Artifact
64-
uses: actions/upload-artifact@v1
91+
jb build lectures --path-output ./ -n -W --keep-going
92+
- name: Upload Execution Reports
93+
uses: actions/upload-artifact@v2
94+
if: failure()
6595
with:
66-
name: _build
67-
path: _build
96+
name: execution-reports
97+
path: _build/html/reports
6898
- name: Preview Deploy to Netlify
6999
uses: nwtgck/actions-netlify@v1.1
70100
with:
71101
publish-dir: '_build/html/'
72-
production-branch: master
102+
production-branch: main
73103
github-token: ${{ secrets.GITHUB_TOKEN }}
74104
deploy-message: "Preview Deploy from GitHub Actions"
75105
env:

‎.github/workflows/publish.yml‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,35 @@
1-
name: Build & Publish to GH-PAGES
1+
name: Build & Publish to GH Pages
22
on:
33
push:
44
tags:
55
- 'publish*'
66
jobs:
7+
deploy-runner:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: iterative/setup-cml@v1
11+
- uses: actions/checkout@v3
12+
with:
13+
ref: ${{ github.event.pull_request.head.sha }}
14+
- name: Deploy runner on EC2
15+
env:
16+
REPO_TOKEN: ${{ secrets.QUANTECON_SERVICES_PAT }}
17+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
18+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
19+
run: |
20+
cml runner launch \
21+
--cloud=aws \
22+
--cloud-region=us-west-2 \
23+
--cloud-type=p3.2xlarge \
24+
--labels=cml-gpu \
25+
--cloud-hdd-size=40
726
publish:
827
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
9-
runs-on: ubuntu-latest
28+
needs: deploy-runner
29+
runs-on: [self-hosted, cml-gpu]
30+
container:
31+
image: docker://nvidia/cuda:11.2.1-devel-ubuntu20.04
32+
options: --gpus all
1033
steps:
1134
- name: Checkout
1235
uses: actions/checkout@v2
@@ -19,37 +42,44 @@ jobs:
1942
python-version: 3.9
2043
environment-file: environment.yml
2144
activate-environment: quantecon
22-
- name: Install latex dependencies
45+
- name: Install Jax and Upgrade CUDA
46+
shell: bash -l {0}
47+
run: |
48+
pip install --upgrade "jax[cuda]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
49+
nvidia-smi
50+
- name: Install Supporting Packages
2351
run: |
24-
sudo apt-get -qq update
25-
sudo apt-get install -y \
52+
apt-get -qq update
53+
export DEBIAN_FRONTEND=noninteractive
54+
apt-get install -y tzdata
55+
apt-get install -y git
56+
apt-get install -y \
2657
texlive-latex-recommended \
2758
texlive-latex-extra \
2859
texlive-fonts-recommended \
2960
texlive-fonts-extra \
3061
texlive-xetex \
3162
latexmk \
32-
xindy \
33-
dvipng \
34-
cm-super
63+
xindy
3564
- name: Display Conda Environment Versions
3665
shell: bash -l {0}
3766
run: conda list
3867
- name: Display Pip Versions
3968
shell: bash -l {0}
4069
run: pip list
70+
# Download Build Cache from cache.yml
4171
- name: Download "build" folder (cache)
4272
uses: dawidd6/action-download-artifact@v2
4373
with:
4474
workflow: cache.yml
4575
branch: main
4676
name: build-cache
4777
path: _build
48-
# Build Assets (Download Notebooks and PDF via LaTeX)
78+
# Build Assets (Download Notebooks, PDF via LaTeX)
4979
- name: Build PDF from LaTeX
5080
shell: bash -l {0}
5181
run: |
52-
jb build lectures --builder pdflatex --path-output ./ -n --keep-going
82+
jb build lectures --builder pdflatex --path-output ./ -n -W --keep-going
5383
- name: Copy LaTeX PDF for GH-PAGES
5484
shell: bash -l {0}
5585
run: |
@@ -58,7 +88,7 @@ jobs:
5888
- name: Build Download Notebooks (sphinx-tojupyter)
5989
shell: bash -l {0}
6090
run: |
61-
jb build lectures --path-output ./ --builder=custom --custom-builder=jupyter
91+
jb build lectures --path-output ./ --builder=custom --custom-builder=jupyter -n -W --keep-going
6292
zip -r download-notebooks.zip _build/jupyter
6393
- uses: actions/upload-artifact@v2
6494
with:
@@ -69,11 +99,11 @@ jobs:
6999
run: |
70100
mkdir -p _build/html/_notebooks
71101
cp -u _build/jupyter/*.ipynb _build/html/_notebooks
72-
# Build HTML (Website)
102+
# Final Build of HTML (with assets)
73103
- name: Build HTML
74104
shell: bash -l {0}
75105
run: |
76-
jb build lectures --path-output ./
106+
jb build lectures --path-output ./ -n -W --keep-going
77107
- name: Deploy website to gh-pages
78108
uses: peaceiris/actions-gh-pages@v3
79109
with:
@@ -87,14 +117,18 @@ jobs:
87117
cp -a _notebook_repo/. _build/lecture-python.notebooks
88118
cp _build/jupyter/*.ipynb _build/lecture-python.notebooks
89119
ls -a _build/lecture-python.notebooks
90-
- name: Commit latest notebooks to lecture-python.notebooks
91-
uses: cpina/github-action-push-to-another-repository@v1.3
120+
- name: Commit notebooks to lecture-python.notebooks
121+
shell: bash -l {0}
92122
env:
93-
API_TOKEN_GITHUB: ${{ secrets.QUANTECON_SERVICES_PAT }}
94-
with:
95-
source-directory: '_build/lecture-python.notebooks/'
96-
destination-repository-username: 'QuantEcon'
97-
destination-repository-name: 'lecture-python.notebooks'
98-
commit-message: 'auto publishing updates to notebooks'
99-
destination-github-username: 'quantecon-services'
100-
user-email: services@quantecon.org
123+
QE_SERVICES_PAT: ${{ secrets.QUANTECON_SERVICES_PAT }}
124+
run: |
125+
git clone https://quantecon-services:$QE_SERVICES_PAT@github.com/quantecon/lecture-python.notebooks
126+
127+
cp _build/lecture-python.notebooks/*.ipynb lecture-python.notebooks
128+
129+
cd lecture-python.notebooks
130+
git config user.name "QuantEcon Services"
131+
git config user.email "admin@quantecon.org"
132+
git add *.ipynb
133+
git commit -m "auto publishing updates to notebooks"
134+
git push origin master

0 commit comments

Comments
 (0)