Skip to content

Commit 0aa1a53

Browse files
mkoeppemkoeppe
committed
.github/workflows/ci-sage.yml: New
1 parent 7dc31d3 commit 0aa1a53

File tree

1 file changed

+258
-0
lines changed

1 file changed

+258
-0
lines changed

‎.github/workflows/ci-sage.yml‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
name: Run Sage CI for Linux/Cygwin/macOS
2+
3+
## This GitHub Actions workflow provides:
4+
##
5+
## - portability testing, by building and testing this project on many platforms
6+
## (Linux variants and Cygwin), each with two configurations (installed packages),
7+
##
8+
## - continuous integration, by building and testing other software
9+
## that depends on this project.
10+
##
11+
## It runs on every pull request and push of a tag to the GitHub repository.
12+
##
13+
## The testing can be monitored in the "Actions" tab of the GitHub repository.
14+
##
15+
## After all jobs have finished (or are canceled) and a short delay,
16+
## tar files of all logs are made available as "build artifacts".
17+
##
18+
## This GitHub Actions workflow uses the portability testing framework
19+
## of SageMath (https://www.sagemath.org/). For more information, see
20+
## https://doc.sagemath.org/html/en/developer/portability_testing.html
21+
22+
## The workflow consists of two jobs:
23+
##
24+
## - First, it builds a source distribution of the project
25+
## and generates a script "update-pkgs.sh". It uploads them
26+
## as a build artifact named upstream.
27+
##
28+
## - Second, it checks out a copy of the SageMath source tree.
29+
## It downloads the upstream artifact and replaces the project's
30+
## package in the SageMath distribution by the newly packaged one
31+
## from the upstream artifact, by running the script "update-pkgs.sh".
32+
## Then it builds a small portion of the Sage distribution.
33+
##
34+
## Many copies of the second step are run in parallel for each of the tested
35+
## systems/configurations.
36+
37+
#on: [push, pull_request]
38+
39+
on:
40+
pull_request:
41+
types: [opened, synchronize]
42+
push:
43+
tags:
44+
- '*'
45+
46+
env:
47+
# Ubuntu packages to install so that the project's "make dist" can succeed
48+
DIST_PREREQ:
49+
# Name of this project in the Sage distribution
50+
SPKG: flint
51+
# Sage distribution packages to build
52+
TARGETS_PRE: build/make/Makefile
53+
TARGETS: SAGE_CHECK=yes flint
54+
TARGETS_OPTIONAL: SAGE_CHECK=warn arb
55+
# Standard setting: Test the current beta release of Sage:
56+
SAGE_REPO: sagemath/sage
57+
SAGE_REF: develop
58+
# Temporarily test on the branch from sage ticket https://trac.sagemath.org/ticket/34102
59+
# FLINT 2.9 upgrade ticket; this is a no-op when merged in Sage develop.
60+
SAGE_TRAC_GIT: https://github.com/sagemath/sagetrac-mirror.git
61+
SAGE_TICKET: 34102
62+
#REMOVE_PATCHES: "*"
63+
64+
jobs:
65+
66+
dist:
67+
runs-on: ubuntu-latest
68+
steps:
69+
- name: Check out ${{ env.SPKG }}
70+
uses: actions/checkout@v2
71+
with:
72+
path: build/pkgs/${{ env.SPKG }}/src
73+
- name: Install prerequisites
74+
run: |
75+
sudo DEBIAN_FRONTEND=noninteractive apt-get update
76+
sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ
77+
if: env.DIST_PREREQ != ''
78+
- name: Run make dist, prepare upstream artifact
79+
run: |
80+
(cd build/pkgs/${{ env.SPKG }}/src && (cd .. && tar czf - src) > arb-git.tar.gz) \
81+
&& mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \
82+
&& echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=optional" > upstream/update-pkgs.sh \
83+
&& if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \
84+
&& ls -l upstream/
85+
- uses: actions/upload-artifact@v2
86+
with:
87+
path: upstream
88+
name: upstream
89+
90+
cygwin:
91+
uses: sagemath/sagetrac-mirror/.github/workflows/cygwin.yml@u/mkoeppe/ci_cygwin__refactor_using_reusable_workflows
92+
with:
93+
# FIXME: duplicated from env.TARGETS
94+
targets: SAGE_CHECK=yes flint
95+
prefix: /opt/sage-flint
96+
sage_repo: sagemath/sage
97+
sage_ref: develop
98+
upstream_artifact: upstream
99+
sage_trac_git: https://github.com/sagemath/sagetrac-mirror.git
100+
sage_trac_ticket: 34102
101+
needs: [dist]
102+
103+
docker:
104+
runs-on: ubuntu-latest
105+
needs: [dist]
106+
strategy:
107+
fail-fast: false
108+
max-parallel: 32
109+
matrix:
110+
tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, raspbian-buster-armhf]
111+
tox_packages_factor: [minimal, standard]
112+
env:
113+
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
114+
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
115+
DOCKER_TARGETS: configured with-targets with-targets-optional
116+
steps:
117+
- name: Check out SageMath
118+
uses: actions/checkout@v2
119+
with:
120+
repository: ${{ env.SAGE_REPO }}
121+
ref: ${{ env.SAGE_REF }}
122+
fetch-depth: 2000
123+
if: env.SAGE_REPO != ''
124+
- name: Check out git-trac-command
125+
uses: actions/checkout@v2
126+
with:
127+
repository: sagemath/git-trac-command
128+
path: git-trac-command
129+
if: env.SAGE_TRAC_GIT != ''
130+
- name: Check out SageMath from trac.sagemath.org
131+
shell: bash {0}
132+
run: |
133+
git config --global user.email "ci-sage@example.com"
134+
git config --global user.name "ci-sage workflow"
135+
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
136+
if: env.SAGE_TRAC_GIT != ''
137+
- uses: actions/download-artifact@v2
138+
with:
139+
path: upstream
140+
name: upstream
141+
- name: Install test prerequisites
142+
run: |
143+
sudo DEBIAN_FRONTEND=noninteractive apt-get update
144+
sudo DEBIAN_FRONTEND=noninteractive apt-get install tox python3-setuptools
145+
- name: Update Sage packages from upstream artifact
146+
run: |
147+
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
148+
- name: Configure and build Sage distribution within a Docker container
149+
run: |
150+
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
151+
- name: Copy logs from the Docker image or build container
152+
run: |
153+
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
154+
cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
155+
if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
156+
if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
157+
if: always()
158+
- uses: actions/upload-artifact@v2
159+
with:
160+
path: artifacts
161+
name: ${{ env.LOGS_ARTIFACT_NAME }}
162+
if: always()
163+
- name: Print out logs for immediate inspection
164+
# and markup the output with GitHub Actions logging commands
165+
run: |
166+
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
167+
if: always()
168+
- name: Push Docker images
169+
run: |
170+
if [ -f .tox/$TOX_ENV/Dockertags ]; then
171+
TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
172+
if [ -z "$TOKEN" ]; then
173+
TOKEN="${{ secrets.GITHUB_TOKEN }}"
174+
fi
175+
echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
176+
for a in $(cat .tox/$TOX_ENV/Dockertags); do
177+
FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a
178+
docker tag $a $FULL_TAG
179+
echo Pushing $FULL_TAG
180+
docker push $FULL_TAG
181+
done || echo "(Ignoring errors)"
182+
fi
183+
if: always()
184+
185+
macos:
186+
187+
runs-on: macos-latest
188+
strategy:
189+
fail-fast: false
190+
max-parallel: 4
191+
matrix:
192+
os: [ macos-10.15, macos-11.0 ]
193+
tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos]
194+
tox_packages_factor: [minimal, standard]
195+
xcode_version_factor: [11.7, default, 12.3]
196+
197+
needs: [dist]
198+
199+
env:
200+
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
201+
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
202+
DOCKER_TARGETS: configured with-targets with-targets-optional
203+
204+
steps:
205+
206+
- name: Select Xcode version
207+
run: |
208+
if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
209+
- name: Check out SageMath
210+
uses: actions/checkout@v2
211+
with:
212+
repository: ${{ env.SAGE_REPO }}
213+
ref: ${{ env.SAGE_REF }}
214+
fetch-depth: 2000
215+
if: env.SAGE_REPO != ''
216+
- name: Check out git-trac-command
217+
uses: actions/checkout@v2
218+
with:
219+
repository: sagemath/git-trac-command
220+
path: git-trac-command
221+
if: env.SAGE_TRAC_GIT != ''
222+
- name: Check out SageMath from trac.sagemath.org
223+
shell: bash {0}
224+
run: |
225+
git config --global user.email "ci-sage@example.com"
226+
git config --global user.name "ci-sage workflow"
227+
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
228+
if: env.SAGE_TRAC_GIT != ''
229+
- uses: actions/download-artifact@v2
230+
with:
231+
path: upstream
232+
name: upstream
233+
- name: Update Sage packages from upstream artifact
234+
run: |
235+
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff)
236+
237+
- name: Install test prerequisites
238+
run: |
239+
brew install tox
240+
- name: Build and test with tox
241+
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
242+
# For doctesting, we use a lower parallelization to avoid timeouts.
243+
run: |
244+
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
245+
- name: Prepare logs artifact
246+
run: |
247+
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
248+
if: always()
249+
- uses: actions/upload-artifact@v1
250+
with:
251+
path: artifacts
252+
name: ${{ env.LOGS_ARTIFACT_NAME }}
253+
if: always()
254+
- name: Print out logs for immediate inspection
255+
# and markup the output with GitHub Actions logging commands
256+
run: |
257+
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
258+
if: always()

0 commit comments

Comments
 (0)