## How to build tests and examples
./autogen.sh
./configure
make
## How to run CD tests:
for dev in $(ibstat -l); do
hca="${dev}:1";
GTEST_SHUFFLE=1 IBV_TEST_DEV=${hca} tests/ibv_cc_test;
done
## How to run CD tests w/ valgrind
valgrind --tool=memcheck --leak-check=full --track-origins=yes tests/ibv_cc_test
## Running the tests
The 'recursive doubling barrier' and the 'ping pong latency test' can be run launched via 'mpirun' as follows:
mpirun -np $np ./examples/ibv_cc_barrier --debug=0 --size=8 --np=$np --iters=1000 --warmup=0 --test=barrier
mpirun -np $np ./examples/ibv_cc_barrier --debug=0 --size=8 --np=$np --iters=1000 --warmup=0 --test=latency
Recursive Knomial barrier on RoCE with managed send (MQ=1) with knomial radix 2 (MPI used only for connection data exchange):
N=2; PPN=8; mpirun -n $((N*PPN)) -hostfile ./hosts --display-map --npernode $PPN --bind-to core -mca pml ob1 -mca btl self,sm,tcp -x CC_IB_DEV=mlx4_0 -x CC_IB_PORT=1 -x CC_RADIX=2 -x CC_USE_MQ=1 -x CC_ROCE=1 install/bin/ibv_cc_barrier --size=128 --np=$((N*PPN)) --test=barrier_rk --debug=1 --iters=5000 --warmup=1000 --check=0
There are two available control environment variables:
CC_IB_DEV - specifies the device to be used (e.g. export CC_IB_DEV=mlx5_0)
CC_IB_PORT - specifies the port to be used (e.g. export CC_IB_PORT=1)
Running the 'cc2_pingpong test'
-----------------------------------
# this test assumes 4 clients connect to the server.
export EXE_RDMA=/full/path/to/cc2_rdma_pingpong
export EXE_SEND=/full/path/to/cc2_send_pingpong
SERVER=server_name_here
HCA=mlx5_0
RDMA-based
Server: taskset -c 0 $EXE_RDMA -d mlx4_0 -s 8 -n 2000
Client: for i in 0 2 4 6 ; do eval "taskset -c $i $EXE_RDMA -d $HCA -s 8 -n 2000 $SERVER &" ;done
SEND-based
Server: taskset -c 0 $EXE_SEND -d mlx4_0 -s 8 -n 2000
Client: for i in 0 2 4 6 ; do eval "taskset -c $i $EXE_SEND -d $HCA -s 8 -n 2000 $SERVER &" ;done