running a test with valgrind is broken if we pass flags to mpiexec?
on SUMMIT, I configure --with-mpiexec=jsrun -g1
. When running a test using VALGRIND=1, I get this
bash-4.2$ make -f $PETSC_DIR/gmakefile.test vec_vec_tests-ex23_cuda VALGRIND=1 V=1
/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.3.1.2-20200121-awz2q5brde7wgdqqw4ugalrkukeub4eb/bin/mpicc -c -g -Og -mcpu=power9 -fPIC -fopenmp -I/ccs/home/szampini/Devel/petsc/include -I/ccs/home/szampini/Devel/petsc/arch-h2opus-dbg/include -I/sw/summit/essl/6.2.1-0/essl/6.2/include -I/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/hdf5-1.10.4-qj7qoumocmjj2xkwbwotazo22sugeq5o/include -I/sw/summit/cuda/10.2.89/include -MMD -MP /ccs/home/szampini/Devel/petsc/src/vec/vec/tests/ex23.c -o /gpfs/alpine/proj-shared/csc314/szampini/petsc/vec/vec/tests/ex23.o
/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.3.1.2-20200121-awz2q5brde7wgdqqw4ugalrkukeub4eb/bin/mpicc -fopenmp -fopenmp -g -Og -mcpu=power9 -fPIC -fopenmp -o /gpfs/alpine/proj-shared/csc314/szampini/petsc/vec/vec/tests/ex23 /gpfs/alpine/proj-shared/csc314/szampini/petsc/vec/vec/tests/ex23.o -Wl,-rpath,/ccs/home/szampini/Devel/petsc/arch-h2opus-dbg/lib -L/ccs/home/szampini/Devel/petsc/arch-h2opus-dbg/lib -Wl,-rpath,/sw/summit/essl/6.2.1-0/essl/6.2/lib64 -L/sw/summit/essl/6.2.1-0/essl/6.2/lib64 -Wl,-rpath,/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/netlib-lapack-3.8.0-wcabdyqhdi5rooxbkqa6x5d7hxyxwdkm/lib64 -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/netlib-lapack-3.8.0-wcabdyqhdi5rooxbkqa6x5d7hxyxwdkm/lib64 -Wl,-rpath,/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/hdf5-1.10.4-qj7qoumocmjj2xkwbwotazo22sugeq5o/lib -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/hdf5-1.10.4-qj7qoumocmjj2xkwbwotazo22sugeq5o/lib -Wl,-rpath,/sw/summit/cuda/10.2.89/lib64 -L/sw/summit/cuda/10.2.89/lib64 -Wl,-rpath,/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.3.1.2-20200121-awz2q5brde7wgdqqw4ugalrkukeub4eb/lib -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.3.1.2-20200121-awz2q5brde7wgdqqw4ugalrkukeub4eb/lib -Wl,-rpath,/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib/gcc/powerpc64le-none-linux-gnu/6.4.0 -L/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib/gcc/powerpc64le-none-linux-gnu/6.4.0 -Wl,-rpath,/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib/gcc -L/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib/gcc -Wl,-rpath,/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib64 -L/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib64 -Wl,-rpath,/autofs/nccs-svm1_sw/summit/.swci/0-core/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/valgrind-3.14.0-6tdksrkyyvtn77cypt4g5jp7m5tr4eaq/lib -L/autofs/nccs-svm1_sw/summit/.swci/0-core/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/valgrind-3.14.0-6tdksrkyyvtn77cypt4g5jp7m5tr4eaq/lib -Wl,-rpath,/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/darshan-runtime-3.1.7-cnvxicgf5j4ap64qi6v5gxp67hmrjz43/lib -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/darshan-runtime-3.1.7-cnvxicgf5j4ap64qi6v5gxp67hmrjz43/lib -Wl,-rpath,/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib -L/autofs/nccs-svm1_sw/summit/gcc/6.4.0/lib -lpetsc -lessl -llapacke -llapack -lessl -lhdf5_hl -lhdf5 -lm -lcuda -lcudart -lcufft -lcublas -lcusparse -lcusolver -lcurand -lX11 -lstdc++ -ldl -lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm -lgfortran -lm -lgfortran -lm -lgcc_s -lpthread -lstdc++ -ldl
Using MAKEFLAGS: V=1 VALGRIND=1
/gpfs/alpine/proj-shared/csc314/szampini/petsc/vec/vec/tests/runex23_cuda.sh -v -V
not ok vec_vec_tests-ex23_cuda # Error code: 64
# Error, invalid argument: valgrind
#
# Usage: jsrun [OPTION...] [<command>]
# Try `jsrun --help' or `jsrun --usage' for more information.
ok vec_vec_tests-ex23_cuda # SKIP Command failed so no diff
If I print the command, I get this
jsrun -g1 -n valgrind -q --tool=memcheck --leak-check=yes --num-callers=20 --track-origins=yes --suppressions=/ccs/home/szampini/Devel/petsc/lib/petsc/bin/maint/petsc-val.supp --error-exitcode=10 2 ../ex23 -vec_type cuda