Skip to content

Commit f07d66a

Browse files
udesouLuis Eduardo de Souza Amorimqinsoon
authored
Adding our own CI (#216)
Setting up our CI to run on `master`. It has been rewritten to use make. This PR also fixes and sets up `cargo clippy` to run as one of the checks. --------- Co-authored-by: Luis Eduardo de Souza Amorim <eduardo@rat.moma> Co-authored-by: Yi Lin <qinsoon@gmail.com>
1 parent fd0e7e8 commit f07d66a

29 files changed

+677
-230
lines changed

‎.github/scripts/Make.user‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
FORCE_ASSERTIONS=1
2+
LLVM_ASSERTIONS=1

‎.github/scripts/ci-build.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
set -xe
2+
3+
. $(dirname "$0")/common.sh
4+
5+
# Need a build_type argument
6+
if [ $# -eq 0 ]
7+
then
8+
echo "No arguments supplied"
9+
exit 1
10+
fi
11+
# debug or release
12+
build_type=$1
13+
# plan to use
14+
plan=$2
15+
# moving vs non-moving
16+
is_moving=$3
17+
18+
# helloworld.jl
19+
HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl
20+
21+
# build MMTk
22+
build_args=""
23+
if [ "$build_type" == "release" ]; then
24+
build_args=$build_args" --release"
25+
fi
26+
27+
plan_feature=${plan,,}
28+
moving_feature=${is_moving,,}
29+
30+
if [ "$is_moving" == "moving" ]; then
31+
MOVING=1
32+
else
33+
MOVING=0
34+
fi
35+
36+
cd $MMTK_JULIA_DIR
37+
38+
MMTK_MOVING=$MOVING make $build_type
39+
40+
cd $JULIA_PATH
41+
42+
# Clean first
43+
make cleanall
44+
# Build
45+
cp $BINDING_PATH/.github/scripts/Make.user $JULIA_PATH/
46+
MMTK_PLAN=$plan MMTK_BUILD=$build_type make
47+
# Run hello world
48+
$JULIA_PATH/julia $HELLO_WORLD_JL

‎.github/scripts/ci-checkout.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
set -ex
2+
3+
. $(dirname "$0")/common.sh
4+
5+
# We may later allow setting up a specific version of Julia using comments
6+
# in the PR, but for now we just use the latest master from JuliaLang
7+
JULIA_URL=https://github.com/$1.git
8+
JULIA_VERSION=$2
9+
10+
rm -rf $JULIA_PATH
11+
git clone $JULIA_URL $JULIA_PATH
12+
git -C $JULIA_PATH checkout $JULIA_VERSION

‎.github/scripts/ci-ffi.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
set -ex
2+
3+
. $(dirname "$0")/common.sh
4+
5+
pushd $MMTK_JULIA_DIR
6+
7+
make regen-bindgen-ffi
8+
9+
popd
10+

‎.github/scripts/ci-setup.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
set -xe
2+
3+
. $(dirname "$0")/common.sh
4+
5+
rustup toolchain install $RUSTUP_TOOLCHAIN
6+
rustup target add i686-unknown-linux-gnu --toolchain $RUSTUP_TOOLCHAIN
7+
rustup component add clippy --toolchain $RUSTUP_TOOLCHAIN
8+
rustup component add rustfmt --toolchain $RUSTUP_TOOLCHAIN
9+
rustup override set $RUSTUP_TOOLCHAIN

‎.github/scripts/ci-style.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
set -xe
2+
3+
. $(dirname "$0")/common.sh
4+
5+
export RUSTFLAGS="-D warnings"
6+
7+
pushd $BINDING_PATH/mmtk
8+
9+
cargo clippy
10+
cargo clippy --release
11+
12+
cargo fmt -- --check
13+
popd

‎.github/scripts/ci-test-LinearAlgebra.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Running LinearAlgebra as a separate item
2+
# Given it takes on average more than 2.5h to run
3+
4+
set -e
5+
6+
. $(dirname "$0")/common.sh
7+
8+
export MMTK_MAX_HSIZE_G=8
9+
total_mem=$(free -m | awk '/^Mem:/ {print $2}')
10+
mem_threshold=512 # use 0.5Gb as a threshold for the max rss based on the total free memory
11+
total_mem_restricted=$((total_mem- mem_threshold))
12+
num_workers=2
13+
export JULIA_TEST_MAXRSS_MB=$((total_mem_restricted/ num_workers))
14+
15+
echo "-> Run single threaded"
16+
ci_run_jl_test "LinearAlgebra" 2

‎.github/scripts/ci-test-gc-core.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
set -e
2+
3+
# run tests that are more specific to the GC
4+
# this list was built intuitively based on tests that
5+
# call the gc manually, have finalizers, have multiple threads, etc.
6+
7+
. $(dirname "$0")/common.sh
8+
9+
declare -a test_names=(
10+
"gc" # Julia's GC specific testset
11+
"threads" # Tests for multithreading (with some GC interaction)
12+
"cmdlineargs" # Has tests that spawn other processes
13+
"compiler" # Some tests for allocation, compiled code checks
14+
"misc" # set of miscelaneous tests, include finalizers, eg
15+
"core" # should have the core of the Julia code
16+
"dict" # tests for weak references
17+
)
18+
19+
for i in "${test_names[@]}"
20+
do
21+
# echo "Token: '$i'"
22+
test=`sed 's/\"\(.*\)\"/\1/' <<< $i`
23+
if [[ ! -z "$test" ]]; then
24+
echo $test
25+
26+
echo "-> Run"
27+
ci_run_jl_test $test
28+
fi
29+
done

‎.github/scripts/ci-test-other.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
set -xe
2+
3+
. $(dirname "$0")/common.sh
4+
5+
# Get all the tests
6+
CHOOSE_TESTS_JL_PATH=$JULIA_PATH/test/choosetests.jl
7+
CHOOSE_TESTS_JL_CONTENT=`cat $CHOOSE_TESTS_JL_PATH`
8+
9+
REGEX_PATTERN='.*const TESTNAMES = \[([^\[]*)^\].*'
10+
11+
if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
12+
RAW_TEST_NAMES=${BASH_REMATCH[1]}
13+
14+
readarray -td, test_names <<< "$RAW_TEST_NAMES"
15+
declare test_names
16+
17+
for i in "${test_names[@]}"
18+
do
19+
# echo "Token: '$i'"
20+
test=`sed 's/\"\(.*\)\"/\1/' <<< $i`
21+
if [[ ! -z "$test" ]]; then
22+
echo $test
23+
24+
# Should we skip some tests?
25+
# Ignore stdlib tests for now -- we run stdlib tests separately
26+
if [[ $test =~ "stdlib" ]]; then
27+
echo "-> Skip stdlib"
28+
continue
29+
fi
30+
31+
echo "-> Run"
32+
ci_run_jl_test $test
33+
fi
34+
done
35+
else
36+
echo "Cannot find TESTNAMES in $CHOOSE_TESTS_JL_PATH"
37+
exit 1
38+
fi

‎.github/scripts/ci-test-patching.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
set -xe
2+
3+
. $(dirname "$0")/common.sh
4+
5+
# The list of tests/checks that we need to skip. They are either not suitable for Julia-MMTk, or
6+
# not supported at this moment.
7+
# Each line is a pattern of the test to match (we add skip=true to the end of those lines), and the test file path
8+
# * Pattern ends with $ so we won't append 'skip=true' multiple times
9+
declare -a tests_to_skip=(
10+
# Ignore the entire libgit2.jl -- there are too many possible network related issues to run this test
11+
# '@test.*$' "$JULIA_PATH/usr/share/julia/stdlib/v1.8/LibGit2/test/libgit2.jl"
12+
13+
# These tests check for the number of stock GC threads (which we set to 0 with mmtk)
14+
'@test string(cpu_threads) ==' "$JULIA_PATH/test/cmdlineargs.jl"
15+
'@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
16+
'@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
17+
'@test read(`$exename --gcthreads=2,1 -e $code`, String) == "3"' "$JULIA_PATH/test/cmdlineargs.jl"
18+
'@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
19+
'@test read(`$exename -e $code`, String) == "3"' "$JULIA_PATH/test/cmdlineargs.jl"
20+
21+
# These tests use the heapsize hint which is not used by mmtk
22+
'@test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M' "$JULIA_PATH/test/cmdlineargs.jl"
23+
'@test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M' "$JULIA_PATH/test/cmdlineargs.jl"
24+
'@test abs(Float64(maxmem)' "$JULIA_PATH/test/cmdlineargs.jl"
25+
26+
# For some reason this fails even with the stock build
27+
'@test n_precompiles <= expected_precompiles' "$JULIA_PATH/stdlib/REPL/test/precompilation.jl"
28+
'@test length(targets) > 1' "$JULIA_PATH/test/precompile.jl"
29+
30+
# rr might not be available in the github runner
31+
'@test success(pipeline(setenv(`$(Base.julia_cmd()) --bug-report=rr-local' "$JULIA_PATH/test/cmdlineargs.jl"
32+
33+
# These tests seem to fail because we set the number of stock GC threads to 0
34+
'jl_setaffinity(1, mask, cpumasksize) == 0' "$JULIA_PATH/test/threads.jl"
35+
'jl_getaffinity(1, mask, cpumasksize) == 0' "$JULIA_PATH/test/threads.jl"
36+
37+
# Skipping these GC tests for now (until we make sure we follow the stats as expected by the stock GC)
38+
'@test !live_bytes_has_grown_too_much' "$JULIA_PATH/test/gc.jl"
39+
'@test any(page_utilization .> 0)' "$JULIA_PATH/test/gc.jl"
40+
41+
# Tests that check the reasons for a full sweep and are specific to stock Julia
42+
'@test reasons\[:FULL_SWEEP_REASON_FORCED_FULL_SWEEP\] >= 1' "$JULIA_PATH/test/gc.jl"
43+
'@test keys(reasons) == Set(Base.FULL_SWEEP_REASONS)' "$JULIA_PATH/test/gc.jl"
44+
45+
# Allocation profiler tests that fail when we inline fastpath allocation
46+
'@test length(\[a for a in prof.allocs if a.type == MyType\]) >= 1' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
47+
'@test length(prof.allocs) >= 1' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
48+
'@test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
49+
'@test length(profile.allocs) >= 2\*NUM_TASKS' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
50+
51+
# Test that expects information from heap snapshot which is currently not available in MMTk
52+
'@test contains(sshot, "redact_this")' "$JULIA_PATH/stdlib/Profile/test/runtests.jl"
53+
54+
# This test checks GC logging
55+
'@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
56+
57+
# These tests check for the number of stock GC threads (which we set to 0 with mmtk)
58+
'@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
59+
'@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
60+
'@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
61+
# This seems to be a regression from upstream when we merge with upstream 43bf2c8.
62+
# The required string int.jl does not appear in the output even if I test with the stock Julia code.
63+
# I do not know what is wrong, but at this point, I dont want to spend time on it.
64+
'@test occursin("int.jl", code)' "$JULIA_PATH/test/cmdlineargs.jl"
65+
)
66+
67+
for (( i=0; i < ${#tests_to_skip[@]}; i+=2 )); do
68+
pattern=${tests_to_skip[i]}
69+
file=${tests_to_skip[i+1]}
70+
sed -i '/'"$pattern"'/ s/@test/@test_skip/' $file
71+
done

0 commit comments

Comments
 (0)