From f382ce43d9b3b54f230516eff6d40628d641c22a Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Thu, 23 Apr 2020 18:26:00 +0200 Subject: [PATCH 01/11] Added nrntraub test --- tests/jenkins/Jenkinsfile | 37 +++++++++++++++++++++++++++++++++++ tests/jenkins/_test_utils.sh | 20 +++++++++++++++++++ tests/jenkins/run_nrntraub.sh | 26 ++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 tests/jenkins/_test_utils.sh create mode 100644 tests/jenkins/run_nrntraub.sh diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 65ce86693..579f8146b 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -132,6 +132,14 @@ pipeline { } } } + stage('nrntraub'){ + steps{ + dir('nrntraub'){ + git url: 'https://github.com/iomaganaris/nrntraub.git' + git branch: 'icei' + } + } + } } } @@ -152,6 +160,11 @@ pipeline { sh 'sh tests/jenkins/nrnivmodl.sh tqperf' } } + stage('nrnivmodl tqperf'){ + steps{ + sh 'sh tests/jenkins/nrnivmodl.sh nrntraub' + } + } stage('nrnivmodl-core testcorenrn AoS'){ steps{ sh 'sh tests/jenkins/nrnivmodl-core.sh testcorenrn AoS' @@ -182,6 +195,16 @@ pipeline { sh 'sh tests/jenkins/nrnivmodl-core.sh tqperf SoA' } } + stage('nrnivmodl-core nrntraub AoS'){ + steps{ + sh 'sh tests/jenkins/nrnivmodl-core.sh nrntraub AoS' + } + } + stage('nrnivmodl-core nrntraub SoA'){ + steps{ + sh 'sh tests/jenkins/nrnivmodl-core.sh nrntraub SoA' + } + } } } @@ -392,6 +415,20 @@ pipeline { } } } + stage('nrntraub'){ + stages{ + stage('neuron + corenrn AoS'){ + steps{ + sh 'sh tests/jenkins/run_nrntraub.sh AoS 4 9' + } + } + stage('neuron + corenrn SoA'){ + steps{ + sh 'sh tests/jenkins/run_corenrn.sh SoA 4 9' + } + } + } + } } } } diff --git a/tests/jenkins/_test_utils.sh b/tests/jenkins/_test_utils.sh new file mode 100644 index 000000000..7e2f53399 --- /dev/null +++ b/tests/jenkins/_test_utils.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +bb5_run() ( + set +x -e + # default partition is interactive. during night use production + hour=`date +%H` + weekday=`date +%u` + if [ "$hour" -ge "19" ] || [ "$hour" -lt "8" ] || [ $weekday -gt 5 ]; then export SALLOC_PARTITION="prod"; fi + + N=${N:-1} + if [ -n "$n" ]; then + SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=$n --exclusive --mem=0" + else + SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=36 --exclusive --mem=0" + fi + + cmd_base="time salloc -N$N $SALLOC_OPTS -Aproj16 --hint=compute_bound -Ccpu|nvme --time 1:00:00 srun dplace " + echo "$cmd_base $@" + $cmd_base "$@" +) diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh new file mode 100644 index 000000000..8d8a3968b --- /dev/null +++ b/tests/jenkins/run_nrntraub.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash + +set -e +source ${JENKINS_DIR:-.}/_env_setup.sh +source ${JENKINS_DIR:-.}/_test_utils.sh + +set -x + +CORENRN_TYPE="$1" +MPI_RANKS="$2" +THREADS="$3" + +cd $WORKSPACE/nrntraub + +# Allocate 1 node +N=1 + +bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc + +export OMP_NUM_THREADS=${THREADS} +n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat + +sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn.sorted +sort -n -k'1,1' -k2 < out36.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_n.sorted + +sdiff -s out_cn.sorted out_n.sorted \ No newline at end of file From 51c79c55fd4ded00528bd54aa7bd66fe6003ced9 Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Thu, 23 Apr 2020 18:29:25 +0200 Subject: [PATCH 02/11] Fixed checking out nrntraub --- tests/jenkins/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 579f8146b..7d395267c 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -135,8 +135,7 @@ pipeline { stage('nrntraub'){ steps{ dir('nrntraub'){ - git url: 'https://github.com/iomaganaris/nrntraub.git' - git branch: 'icei' + git branch: 'icei', url: 'https://github.com/iomaganaris/nrntraub.git' } } } From a7098b6aa3887741337d65128fa4c321ca7b39c4 Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Thu, 23 Apr 2020 18:51:02 +0200 Subject: [PATCH 03/11] Fixed nrnivmodl stage of nrntraub --- tests/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 7d395267c..00b01822e 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -159,7 +159,7 @@ pipeline { sh 'sh tests/jenkins/nrnivmodl.sh tqperf' } } - stage('nrnivmodl tqperf'){ + stage('nrnivmodl nrntraub'){ steps{ sh 'sh tests/jenkins/nrnivmodl.sh nrntraub' } From f9143ce3132e8a8c6b86fcd6b41da23c3240343c Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Thu, 23 Apr 2020 21:04:21 +0200 Subject: [PATCH 04/11] Avoid doing an exclusive allocation --- tests/jenkins/_test_utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/jenkins/_test_utils.sh b/tests/jenkins/_test_utils.sh index 7e2f53399..3cd1c2755 100644 --- a/tests/jenkins/_test_utils.sh +++ b/tests/jenkins/_test_utils.sh @@ -9,9 +9,9 @@ bb5_run() ( N=${N:-1} if [ -n "$n" ]; then - SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=$n --exclusive --mem=0" + SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=$n" else - SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=36 --exclusive --mem=0" + SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=36" fi cmd_base="time salloc -N$N $SALLOC_OPTS -Aproj16 --hint=compute_bound -Ccpu|nvme --time 1:00:00 srun dplace " From e34f5202c13e9adc69e190deb670f69b4d5ce5ef Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 29 Apr 2020 11:03:32 +0200 Subject: [PATCH 05/11] Run nrntraub with corenrn SoA and openmp and AoS pure MPI --- tests/jenkins/run_nrntraub.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index 8d8a3968b..c0c763abf 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -17,8 +17,12 @@ N=1 bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc -export OMP_NUM_THREADS=${THREADS} -n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat +if [ "${CORENRN_TYPE}" = "SoA" ]; then + export OMP_NUM_THREADS=${THREADS} + n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat +elif [ "${CORENRN_TYPE}" = "AoS" ]; then + bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat +fi sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn.sorted sort -n -k'1,1' -k2 < out36.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_n.sorted From 86343c7218a69fa3b1317f9ddfb5e77b906f060d Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 29 Apr 2020 11:51:33 +0200 Subject: [PATCH 06/11] Added --voltage 1000 in corenrn to get same results --- tests/jenkins/run_nrntraub.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index c0c763abf..ecaa3777d 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -19,9 +19,9 @@ bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc if [ "${CORENRN_TYPE}" = "SoA" ]; then export OMP_NUM_THREADS=${THREADS} - n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat + n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 elif [ "${CORENRN_TYPE}" = "AoS" ]; then - bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat + bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 fi sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn.sorted From e591d018a23fb60ebb840dd36b40319ad6709b5d Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 29 Apr 2020 14:34:20 +0200 Subject: [PATCH 07/11] Fix SoA nrnrtraub run --- tests/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 00b01822e..d5300e8f2 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -423,7 +423,7 @@ pipeline { } stage('neuron + corenrn SoA'){ steps{ - sh 'sh tests/jenkins/run_corenrn.sh SoA 4 9' + sh 'sh tests/jenkins/run_nrntraub.sh SoA 4 9' } } } From 77f6357b2de4f3d249cfe1ae6232ff8ce95c378f Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 29 Apr 2020 14:59:32 +0200 Subject: [PATCH 08/11] Run neuron test only once --- tests/jenkins/Jenkinsfile | 9 +++++++-- tests/jenkins/run_neuron.sh | 6 ++++++ tests/jenkins/run_nrntraub.sh | 10 +++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index d5300e8f2..cf3ad601e 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -416,12 +416,17 @@ pipeline { } stage('nrntraub'){ stages{ - stage('neuron + corenrn AoS'){ + stage('neuron'){ + steps{ + sh 'sh tests/jenkins/run_neuron.sh nrntraub nrntraub 36' + } + } + stage('corenrn AoS'){ steps{ sh 'sh tests/jenkins/run_nrntraub.sh AoS 4 9' } } - stage('neuron + corenrn SoA'){ + stage('corenrn SoA'){ steps{ sh 'sh tests/jenkins/run_nrntraub.sh SoA 4 9' } diff --git a/tests/jenkins/run_neuron.sh b/tests/jenkins/run_neuron.sh index ed1fe2141..7bdf3e4e6 100755 --- a/tests/jenkins/run_neuron.sh +++ b/tests/jenkins/run_neuron.sh @@ -2,6 +2,7 @@ set -e source ${JENKINS_DIR:-.}/_env_setup.sh +source ${JENKINS_DIR:-.}/_test_utils.sh set -x TEST_DIR="$1" @@ -24,6 +25,11 @@ elif [ "${TEST_DIR}" = "tqperf" ]; then mkdir ${TEST} mpirun -n ${MPI_RANKS} ./x86_64/special -c tstop=50 run.hoc -mpi cat spk000.dat | sort -k 1n,1n -k 2n,2n > ${TEST}/out_nrn_${TEST}.spk +elif [ "${TEST_DIR}" = "nrntraub" ]; then + # Allocate 1 node + N=1 + n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc + sort -n -k'1,1' -k2 < out36.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_nrn.sorted else echo "Not a valid TEST" exit 1 diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index ecaa3777d..c6fd21682 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -15,16 +15,16 @@ cd $WORKSPACE/nrntraub # Allocate 1 node N=1 -bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc - if [ "${CORENRN_TYPE}" = "SoA" ]; then export OMP_NUM_THREADS=${THREADS} n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 elif [ "${CORENRN_TYPE}" = "AoS" ]; then bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 +else + echo "Not a valid CORENRN_TYPE" + exit 1 fi -sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn.sorted -sort -n -k'1,1' -k2 < out36.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_n.sorted +sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn_${CORENRN_TYPE}.sorted -sdiff -s out_cn.sorted out_n.sorted \ No newline at end of file +sdiff -s out_cn_${CORENRN_TYPE}.sorted out_nrn.sorted From 8cedc1c4db2a09285ea50482df333ab7fc3412ec Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 29 Apr 2020 19:15:56 +0200 Subject: [PATCH 09/11] Set proper number of threads and ranks to run SoA and AoS builds --- tests/jenkins/Jenkinsfile | 2 +- tests/jenkins/run_nrntraub.sh | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index cf3ad601e..81ed65357 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -423,7 +423,7 @@ pipeline { } stage('corenrn AoS'){ steps{ - sh 'sh tests/jenkins/run_nrntraub.sh AoS 4 9' + sh 'sh tests/jenkins/run_nrntraub.sh AoS 36 1' } } stage('corenrn SoA'){ diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index c6fd21682..40ff4f6c8 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -15,15 +15,9 @@ cd $WORKSPACE/nrntraub # Allocate 1 node N=1 -if [ "${CORENRN_TYPE}" = "SoA" ]; then - export OMP_NUM_THREADS=${THREADS} - n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 -elif [ "${CORENRN_TYPE}" = "AoS" ]; then - bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 -else - echo "Not a valid CORENRN_TYPE" - exit 1 -fi +export OMP_NUM_THREADS=${THREADS} + +n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn_${CORENRN_TYPE}.sorted From cd8a0fabe2e1fdb153d02ecdd5419e365f72892d Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 20 May 2020 09:55:12 +0200 Subject: [PATCH 10/11] Changed nrntraub repo and coreneuron command for nrntraub --- tests/jenkins/Jenkinsfile | 2 +- tests/jenkins/run_nrntraub.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 81ed65357..bc7ba5fcd 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -135,7 +135,7 @@ pipeline { stage('nrntraub'){ steps{ dir('nrntraub'){ - git branch: 'icei', url: 'https://github.com/iomaganaris/nrntraub.git' + git branch: 'icei', url: 'https://github.com/pramodk/nrntraub.git' } } } diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index 40ff4f6c8..cec0041e9 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -17,7 +17,7 @@ N=1 export OMP_NUM_THREADS=${THREADS} -n=${MPI_RANKS} bb5_run ./${CORENRN_TYPE}/special-core --mpi -d coredat --voltage=1000 +n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=1 init.hoc sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn_${CORENRN_TYPE}.sorted From 2c25e347a18e22a22a843185940bfd28de12e047 Mon Sep 17 00:00:00 2001 From: Ioannis Magkanaris Date: Wed, 20 May 2020 11:24:34 +0200 Subject: [PATCH 11/11] Fixed execution with threads and added CORENEURONLIB --- tests/jenkins/Jenkinsfile | 4 ++-- tests/jenkins/run_neuron.sh | 5 +++-- tests/jenkins/run_nrntraub.sh | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index bc7ba5fcd..4ea642b2f 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -418,12 +418,12 @@ pipeline { stages{ stage('neuron'){ steps{ - sh 'sh tests/jenkins/run_neuron.sh nrntraub nrntraub 36' + sh 'sh tests/jenkins/run_neuron.sh nrntraub nrntraub 4 9' } } stage('corenrn AoS'){ steps{ - sh 'sh tests/jenkins/run_nrntraub.sh AoS 36 1' + sh 'sh tests/jenkins/run_nrntraub.sh AoS 4 9' } } stage('corenrn SoA'){ diff --git a/tests/jenkins/run_neuron.sh b/tests/jenkins/run_neuron.sh index 7bdf3e4e6..b8c06d50f 100755 --- a/tests/jenkins/run_neuron.sh +++ b/tests/jenkins/run_neuron.sh @@ -8,6 +8,7 @@ set -x TEST_DIR="$1" TEST="$2" MPI_RANKS="$3" +THREADS="$4" cd $WORKSPACE/${TEST_DIR} @@ -28,8 +29,8 @@ elif [ "${TEST_DIR}" = "tqperf" ]; then elif [ "${TEST_DIR}" = "nrntraub" ]; then # Allocate 1 node N=1 - n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 init.hoc - sort -n -k'1,1' -k2 < out36.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_nrn.sorted + n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 -c nthread=${THREADS} init.hoc + sort -n -k'1,1' -k2 < out${MPI_RANKS}.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_nrn.sorted else echo "Not a valid TEST" exit 1 diff --git a/tests/jenkins/run_nrntraub.sh b/tests/jenkins/run_nrntraub.sh index cec0041e9..5bbebfa25 100644 --- a/tests/jenkins/run_nrntraub.sh +++ b/tests/jenkins/run_nrntraub.sh @@ -15,9 +15,9 @@ cd $WORKSPACE/nrntraub # Allocate 1 node N=1 -export OMP_NUM_THREADS=${THREADS} +export CORENEURONLIB=$(pwd)/${CORENRN_TYPE}/libcorenrnmech.so -n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=1 init.hoc +n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=1 -c nthread=${THREADS} init.hoc sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn_${CORENRN_TYPE}.sorted