Skip to content

Commit cae3082

Browse files
tomato42tomato42
committed
Run mutation testing on multiple workers in parallel
1 parent 09ffa0f commit cae3082

File tree

4 files changed

+267
-0
lines changed

4 files changed

+267
-0
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,3 +369,258 @@ jobs:
369369
COVERALLS_SERVICE_NAME: github
370370
run: |
371371
coveralls --finish
372+
373+
mutation-prepare:
374+
name: Prepare job files for the mutation runners
375+
runs-on: ubuntu-latest
376+
steps:
377+
- uses: actions/checkout@v2
378+
if: ${{ !matrix.container }}
379+
with:
380+
fetch-depth: 50
381+
- name: save session objects
382+
uses: actions/cache@v3
383+
with:
384+
path: |
385+
sessions/
386+
key: sessions-${{ github.sha }}
387+
- name: Install cosmic-ray
388+
run: |
389+
pip3 install cosmic-ray
390+
- name: Install dependencies
391+
run: |
392+
sudo apt-get install -y sqlite3
393+
- name: Display Python version
394+
run: python -c "import sys; print(sys.version)"
395+
- name: Create list of mutations
396+
run: |
397+
cosmic-ray init cosmic-ray.toml session.sqlite
398+
- name: Log number of jobs created
399+
run: |
400+
cr-report session.sqlite | tail -n 3
401+
- name: Split up mutations to workers
402+
run: |
403+
cp session.sqlite session-to_del.sqlite
404+
sqlite3 session-to_del.sqlite "$(cat sql/create_to_del.sql)"
405+
mkdir sessions
406+
for i in $(seq 0 19); do
407+
sed "s/%SHARD%/$i/" < sql/shard-db.sql > shard.sql
408+
cp session-to_del.sqlite session-$i.sqlite
409+
sqlite3 session-$i.sqlite "$(cat shard.sql)"
410+
mv session-$i.sqlite sessions/
411+
done
412+
mutation-execute:
413+
name: Execute mutation testing
414+
needs: mutation-prepare
415+
runs-on: ubuntu-latest
416+
strategy:
417+
fail-fast: true
418+
matrix:
419+
include:
420+
- name: 0
421+
- name: 1
422+
- name: 2
423+
- name: 3
424+
- name: 4
425+
- name: 5
426+
- name: 6
427+
- name: 7
428+
- name: 8
429+
- name: 9
430+
- name: 10
431+
- name: 11
432+
- name: 12
433+
- name: 13
434+
- name: 14
435+
- name: 15
436+
- name: 16
437+
- name: 17
438+
- name: 18
439+
- name: 19
440+
steps:
441+
- uses: actions/checkout@v2
442+
if: ${{ !matrix.container }}
443+
with:
444+
fetch-depth: 1
445+
- name: Session objects
446+
uses: actions/cache@v3
447+
with:
448+
path: |
449+
sessions/
450+
key: sessions-${{ github.sha }}
451+
- name: Session done objects
452+
uses: actions/cache@v3
453+
with:
454+
path: |
455+
sessions-done/session-${{ matrix.name }}-done.sqlite
456+
key: sessions-${{ github.sha }}-${{ matrix.name }}-done
457+
- name: Install gmpy2 dependencies
458+
run: sudo apt-get install -y libmpfr-dev libmpc-dev
459+
- name: Install gmpy2
460+
run: pip install gmpy2
461+
- name: Install build dependencies
462+
run: |
463+
pip install -r build-requirements.txt
464+
pip install cosmic-ray
465+
- name: Run mutation testing
466+
run: |
467+
cp sessions/session-${{ matrix.name }}.sqlite session.sqlite
468+
systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite &
469+
cosmic_pid=$!
470+
for i in $(seq 1 10); do
471+
echo $i
472+
sleep 60
473+
done
474+
kill $cosmic_pid
475+
mkdir sessions-done/
476+
cp session.sqlite sessions-done/session-${{ matrix.name }}-done.sqlite
477+
- name: Report executed
478+
run: |
479+
cr-report session.sqlite | tail -n 3
480+
mutation-combine:
481+
name: Combine mutation testing results
482+
needs: mutation-execute
483+
runs-on: ubuntu-latest
484+
steps:
485+
- uses: actions/checkout@v2
486+
if: ${{ !matrix.container }}
487+
with:
488+
fetch-depth: 1
489+
- name: Session done 0 objects
490+
uses: actions/cache@v3
491+
with:
492+
path: |
493+
sessions-done/session-0-done.sqlite
494+
key: sessions-${{ github.sha }}-0-done
495+
- name: Session done 1 objects
496+
uses: actions/cache@v3
497+
with:
498+
path: |
499+
sessions-done/session-1-done.sqlite
500+
key: sessions-${{ github.sha }}-1-done
501+
- name: Session done 2 objects
502+
uses: actions/cache@v3
503+
with:
504+
path: |
505+
sessions-done/session-2-done.sqlite
506+
key: sessions-${{ github.sha }}-2-done
507+
- name: Session done 3 objects
508+
uses: actions/cache@v3
509+
with:
510+
path: |
511+
sessions-done/session-3-done.sqlite
512+
key: sessions-${{ github.sha }}-3-done
513+
- name: Session done 4 objects
514+
uses: actions/cache@v3
515+
with:
516+
path: |
517+
sessions-done/session-4-done.sqlite
518+
key: sessions-${{ github.sha }}-4-done
519+
- name: Session done 5 objects
520+
uses: actions/cache@v3
521+
with:
522+
path: |
523+
sessions-done/session-5-done.sqlite
524+
key: sessions-${{ github.sha }}-5-done
525+
- name: Session done 6 objects
526+
uses: actions/cache@v3
527+
with:
528+
path: |
529+
sessions-done/session-6-done.sqlite
530+
key: sessions-${{ github.sha }}-6-done
531+
- name: Session done 7 objects
532+
uses: actions/cache@v3
533+
with:
534+
path: |
535+
sessions-done/session-7-done.sqlite
536+
key: sessions-${{ github.sha }}-7-done
537+
- name: Session done 8 objects
538+
uses: actions/cache@v3
539+
with:
540+
path: |
541+
sessions-done/session-8-done.sqlite
542+
key: sessions-${{ github.sha }}-8-done
543+
- name: Session done 9 objects
544+
uses: actions/cache@v3
545+
with:
546+
path: |
547+
sessions-done/session-9-done.sqlite
548+
key: sessions-${{ github.sha }}-9-done
549+
- name: Session done 10 objects
550+
uses: actions/cache@v3
551+
with:
552+
path: |
553+
sessions-done/session-10-done.sqlite
554+
key: sessions-${{ github.sha }}-10-done
555+
- name: Session done 11 objects
556+
uses: actions/cache@v3
557+
with:
558+
path: |
559+
sessions-done/session-11-done.sqlite
560+
key: sessions-${{ github.sha }}-11-done
561+
- name: Session done 12 objects
562+
uses: actions/cache@v3
563+
with:
564+
path: |
565+
sessions-done/session-12-done.sqlite
566+
key: sessions-${{ github.sha }}-12-done
567+
- name: Session done 13 objects
568+
uses: actions/cache@v3
569+
with:
570+
path: |
571+
sessions-done/session-13-done.sqlite
572+
key: sessions-${{ github.sha }}-13-done
573+
- name: Session done 14 objects
574+
uses: actions/cache@v3
575+
with:
576+
path: |
577+
sessions-done/session-14-done.sqlite
578+
key: sessions-${{ github.sha }}-14-done
579+
- name: Session done 15 objects
580+
uses: actions/cache@v3
581+
with:
582+
path: |
583+
sessions-done/session-15-done.sqlite
584+
key: sessions-${{ github.sha }}-15-done
585+
- name: Session done 16 objects
586+
uses: actions/cache@v3
587+
with:
588+
path: |
589+
sessions-done/session-16-done.sqlite
590+
key: sessions-${{ github.sha }}-16-done
591+
- name: Session done 17 objects
592+
uses: actions/cache@v3
593+
with:
594+
path: |
595+
sessions-done/session-17-done.sqlite
596+
key: sessions-${{ github.sha }}-17-done
597+
- name: Session done 18 objects
598+
uses: actions/cache@v3
599+
with:
600+
path: |
601+
sessions-done/session-18-done.sqlite
602+
key: sessions-${{ github.sha }}-18-done
603+
- name: Session done 19 objects
604+
uses: actions/cache@v3
605+
with:
606+
path: |
607+
sessions-done/session-19-done.sqlite
608+
key: sessions-${{ github.sha }}-19-done
609+
- name: Install cosmic-ray
610+
run: |
611+
pip3 install cosmic-ray
612+
- name: Install dependencies
613+
run: |
614+
sudo apt-get install -y sqlite3
615+
- name: Combine worker results
616+
run: |
617+
cp sessions-done/session-0-done.sqlite session.sqlite
618+
for i in $(seq 1 19); do
619+
cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite
620+
sqlite3 session.sqlite "$(cat sql/combine.sql)"
621+
done
622+
- name: Report executed
623+
run: |
624+
cr-report session.sqlite | tail -n 3
625+
- name: Check mutation level
626+
run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite

‎sql/combine.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
attach 'session-to_merge.sqlite' as toMerge;
2+
BEGIN;
3+
insert into work_items select * from toMerge.work_items;
4+
insert into mutation_specs select * from toMerge.mutation_specs;
5+
insert into work_results select * from toMerge.work_results;
6+
COMMIT;
7+
detach toMerge;

‎sql/create_to_del.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
create table to_del (job_id VARCHAR NOT NULL, id INTEGER PRIMARY KEY);
2+
insert into to_del select *, ROWID from work_items;

‎sql/shard-db.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
delete from mutation_specs where job_id in (select job_id from to_del where to_del.ID % 20 != %SHARD%);
2+
delete from work_items where job_id in (select job_id from to_del where to_del.ID % 20 != %SHARD%);
3+
drop table to_del;

0 commit comments

Comments
 (0)