Document incompatibility with CUDA <=11.0.3

[hagertnl]

The NVCC wrapper in CUDA versions <= 11.0.3 filters out the required "-DUSE_CUDA" define (in fact, it appears to filter out all defines) when compiling backend-support-tests.c, which results in Spatter not being compiled with the CUDA backend (see last line of the output below).

Current README specifies that CUDA 10.0+ is required, which I believe should be updated to be > 11.0.3. I am not sure 11.0.3 is the last version this is a problem, the next version I have access to is 11.1.1, and that version works. Minimal reproducer attached at the end of this ticket.

[ 33%] Building C object CMakeFiles/spatter.dir/src/backend-support-tests.c.o
/sw/summit/cuda/11.0.3/bin/nvcc -DSG_DATA_TYPE=double -DUSE_CUDA -I/path/to/spatter/src/cuda -isystem /sw/summit/cuda/11.0.3/include -ccbin /sw/summit/gcc/9.3.0-2/bin/gcc --verbose -DSPAT_C_NAME=GNU -O3 -g -I/path/to/spatter/src/include -I/path/to/spatter/external/argtable3 -I/usr/local/cuda/include -DSPAT_C="/sw/summit/cuda/11.0.3/bin/nvcc" -DSPAT_C_VER="9.3.0" -O3 -DNDEBUG -MD -MT CMakeFiles/spatter.dir/src/backend-support-tests.c.o -MF CMakeFiles/spatter.dir/src/backend-support-tests.c.o.d -o CMakeFiles/spatter.dir/src/backend-support-tests.c.o -c /path/to/spatter/src/backend-support-tests.c
#$ _NVVM_BRANCH_=nvvm
#$ _SPACE_=
#$ _CUDART_=cudart
#$ _HERE_=/sw/summit/cuda/11.0.3/bin
#$ _THERE_=/sw/summit/cuda/11.0.3/bin
#$ _TARGET_SIZE_=
#$ _TARGET_DIR_=
#$ _TARGET_DIR_=targets/ppc64le-linux
#$ TOP=/sw/summit/cuda/11.0.3/bin/..
#$ NVVMIR_LIBRARY_DIR=/sw/summit/cuda/11.0.3/bin/../nvvm/libdevice
#$ LD_LIBRARY_PATH=/sw/summit/cuda/11.0.3/bin/../lib:/sw/summit/cuda/11.0.3/lib64:/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-9.3.0/spectrum-mpi-10.4.0.3-20210112-6depextb6p6ulrvmehgtbskbmcsyhtdi/lib:/sw/summit/gcc/9.3.0-2/lib64:/opt/ibm/spectrumcomputing/lsf/10.1.0.11/linux3.10-glibc2.17-ppc64le-csm/lib
#$ PATH=/sw/summit/cuda/11.0.3/bin/../nvvm/bin:/sw/summit/cuda/11.0.3/bin:/sw/summit/nsight-compute/2021.2.1:/sw/summit/xalt/1.2.1/bin:/sw/sources/lsf-tools/2.0/summit/bin:/sw/summit/nsight-systems/2021.3.1.54/bin:/sw/summit/cuda/11.0.3/bin:/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-8.3.1/git-2.36.1-ohqgkmm2pekojsy3jrndrhun344j63ke/bin:/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-8.3.1/cmake-3.23.2-jwqghrfxvfhuozlioasit4bvncwg5yff/bin:/sw/sources/hpss/bin:/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-9.3.0/spectrum-mpi-10.4.0.3-20210112-6depextb6p6ulrvmehgtbskbmcsyhtdi/bin:/sw/summit/gcc/9.3.0-2/bin:/opt/ibm/spectrumcomputing/lsf/10.1.0.11/linux3.10-glibc2.17-ppc64le-csm/etc:/opt/ibm/spectrumcomputing/lsf/10.1.0.11/linux3.10-glibc2.17-ppc64le-csm/bin:/opt/ibm/csm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibm/flightlog/bin:/opt/ibm/jsm/bin:/sw/sources/cgroup_tool/bin:/opt/puppetlabs/bin:/usr/lpp/mmfs/bin
#$ INCLUDES="-I/sw/summit/cuda/11.0.3/bin/../targets/ppc64le-linux/include"
#$ LIBRARIES=  "-L/sw/summit/cuda/11.0.3/bin/../targets/ppc64le-linux/lib/stubs" "-L/sw/summit/cuda/11.0.3/bin/../targets/ppc64le-linux/lib"
#$ CUDAFE_FLAGS=
#$ PTXAS_FLAGS=
#$ "/sw/summit/gcc/9.3.0-2/bin"/gcc -E -x c -D__NVCC__  -O3 -I"/path/to/spatter/src/cuda" -I"/path/to/spatter/src/include" -I"/path/to/spatter/external/argtable3" -I"/usr/local/cuda/include" "-I/sw/summit/cuda/11.0.3/bin/../targets/ppc64le-linux/include"   -isystem "/sw/summit/cuda/11.0.3/include"  -D "SG_DATA_TYPE=double" -D "USE_CUDA" -D "SPAT_C_NAME=GNU" -D "SPAT_C=/sw/summit/cuda/11.0.3/bin/nvcc" -D "SPAT_C_VER=9.3.0" -D "NDEBUG" -D__CUDACC_VER_MAJOR__=11 -D__CUDACC_VER_MINOR__=0 -D__CUDACC_VER_BUILD__=221 -g -gdwarf-2 "/path/to/spatter/src/backend-support-tests.c" -o "/tmp/tmpxft_0006cc3e_00000000-3_backend-support-tests.i"
#$ -- Filter Dependencies -- > CMakeFiles/spatter.dir/src/backend-support-tests.c.o.d
#$ "/sw/summit/gcc/9.3.0-2/bin"/gcc -c -x c -O3 -I"/path/to/spatter/src/cuda" -I"/path/to/spatter/src/include" -I"/path/to/spatter/external/argtable3" -I"/usr/local/cuda/include" "-I/sw/summit/cuda/11.0.3/bin/../targets/ppc64le-linux/include"   -isystem "/sw/summit/cuda/11.0.3/include" -g -gdwarf-2 "/path/to/spatter/src/backend-support-tests.c" -o "CMakeFiles/spatter.dir/src/backend-support-tests.c.o"

Minimal reproducer:

$ cat test.c

#include <stdio.h>

int main(int argc, char **argv) {
#ifdef USE_CUDA
  printf("USE_CUDA was defined\n");
#else
  printf("USE_CUDA was not defined\n");
#endif
  return 0;
}
$ module load cuda/11.1.1
$ nvcc -DUSE_CUDA -MD test.c
$ ./a.out
USE_CUDA was defined
$ module swap cuda/11.1.1 cuda/11.0.3
$ rm ./a.out
$ nvcc -DUSE_CUDA -MD test.c
$ ./a.out
USE_CUDA was not defined

[jyoung3131]

Thanks, Nick. The CUDA backend tests are pretty new, and we've been testing primarily with HPC SDK 23.5 which has a few 11+ versions of CUDA included.

We are talking internally about whether to provide an alternate fix or just to document with a warning for older versions of CUDA.