Assertion `ptr != nullptr' failed

Question

Assertion `ptr != nullptr' failed

Closed this issue a year ago · 6 comments

~/supra_seal$ ./exec.sh

SECTOR_SIZE=32GiB
'[' '' == 512MiB ']'
./build.sh 32GiB
SECTOR_SIZE=-DSECTOR_SIZE_32GiB
'[' 32GiB == 512MiB ']'
CC=cc
CXX=c++
NVCC=nvcc
++++ which nvcc
+++ dirname /usr/local/cuda-12.2/bin/nvcc
++ dirname /usr/local/cuda-12.2/bin
CUDA=/usr/local/cuda-12.2
SPDK=deps/spdk-v22.09
CUDA_ARCH='-arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0'
++ c++ -dM -E -x c++ /dev/null
++ awk '{ if($2=="__cplusplus" && $3<"2017") print "-std=c++17"; }'
CXXSTD=
INCLUDE='-Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include'
CFLAGS='-DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__'
CXXFLAGS='-DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread'
LDFLAGS='-fno-omit-frame-pointer -Wl,-z,relro,-z,now -Wl,-z,noexecstack -fuse-ld=bfd -Ldeps/spdk-v22.09/build/lib -Wl,--whole-archive -Wl,--no-as-needed -lspdk_bdev_malloc -lspdk_bdev_null -lspdk_bdev_nvme -lspdk_bdev_passthru -lspdk_bdev_lvol -lspdk_bdev_raid -lspdk_bdev_error -lspdk_bdev_gpt -lspdk_bdev_split -lspdk_bdev_delay -lspdk_bdev_zone_block -lspdk_blobfs_bdev -lspdk_blobfs -lspdk_blob_bdev -lspdk_lvol -lspdk_blob -lspdk_nvme -lspdk_bdev_ftl -lspdk_ftl -lspdk_bdev_aio -lspdk_bdev_virtio -lspdk_virtio -lspdk_vfio_user -lspdk_accel_ioat -lspdk_ioat -lspdk_scheduler_dynamic -lspdk_env_dpdk -lspdk_scheduler_dpdk_governor -lspdk_scheduler_gscheduler -lspdk_sock_posix -lspdk_event -lspdk_event_bdev -lspdk_bdev -lspdk_notify -lspdk_dma -lspdk_event_accel -lspdk_accel -lspdk_event_vmd -lspdk_vmd -lspdk_event_sock -lspdk_init -lspdk_thread -lspdk_trace -lspdk_sock -lspdk_rpc -lspdk_jsonrpc -lspdk_json -lspdk_util -lspdk_log -Wl,--no-whole-archive deps/spdk-v22.09/build/lib/libspdk_env_dpdk.a -Wl,--whole-archive deps/spdk-v22.09/dpdk/build/lib/librte_bus_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_cryptodev.a deps/spdk-v22.09/dpdk/build/lib/librte_dmadev.a deps/spdk-v22.09/dpdk/build/lib/librte_eal.a deps/spdk-v22.09/dpdk/build/lib/librte_ethdev.a deps/spdk-v22.09/dpdk/build/lib/librte_hash.a deps/spdk-v22.09/dpdk/build/lib/librte_kvargs.a deps/spdk-v22.09/dpdk/build/lib/librte_mbuf.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_net.a deps/spdk-v22.09/dpdk/build/lib/librte_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_power.a deps/spdk-v22.09/dpdk/build/lib/librte_rcu.a deps/spdk-v22.09/dpdk/build/lib/librte_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_telemetry.a deps/spdk-v22.09/dpdk/build/lib/librte_vhost.a -Wl,--no-whole-archive -lnuma -ldl -Ldeps/spdk-v22.09/isa-l/.libs -lisal -pthread -lrt -luuid -lssl -lcrypto -lm -laio'
'[' '!' -d /var/tmp/supra_seal ']'
rm -fr obj
mkdir -p obj
rm -fr bin
mkdir -p bin
mkdir -p deps
'[' '!' -d deps/spdk-v22.09 ']'
'[' '!' -d deps/sppark ']'
'[' '!' -d deps/blst ']'
'[' '!' -d c2/bellperson ']'
cc -c sha/sha_ext_mbx2.S -o obj/sha_ext_mbx2.o
xxd -i poseidon/constants/constants_2
xxd -i poseidon/constants/constants_4
xxd -i poseidon/constants/constants_8
xxd -i poseidon/constants/constants_11
xxd -i poseidon/constants/constants_16
xxd -i poseidon/constants/constants_24
xxd -i poseidon/constants/constants_36
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/sppark/util -o obj/pc1.o -c pc1/pc1.cpp
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -o obj/streaming_node_reader_nvme.o -c nvme/streaming_node_reader_nvme.cpp
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -o obj/ring_t.o -c nvme/ring_t.cpp
c++ -g -O2 -c sealing/sector_parameters.cpp -o obj/sector_parameters.o
nvcc -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -DNO_SPDK -Xcompiler -Wno-subobject-linkage -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -dc pc2/cuda/pc2.cu -o obj/pc2.o
nvcc -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -DNO_SPDK -Xcompiler -Wno-subobject-linkage -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -dlink pc2/cuda/pc2.cu -o obj/pc2_link.o
wait
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -c sealing/supra_seal.cpp -o obj/supra_seal.o -Wno-subobject-linkage
ar rvs obj/libsupraseal.a obj/pc1.o obj/pc2.o obj/pc2_link.o obj/ring_t.o obj/streaming_node_reader_nvme.o obj/supra_seal.o obj/sector_parameters.o obj/sha_ext_mbx2.o
ar: creating obj/libsupraseal.a
a - obj/pc1.o
a - obj/pc2.o
a - obj/pc2_link.o
a - obj/ring_t.o
a - obj/streaming_node_reader_nvme.o
a - obj/supra_seal.o
a - obj/sector_parameters.o
a - obj/sha_ext_mbx2.o
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -D__ADX__ -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -o bin/seal demos/main.cpp -Lobj -lsupraseal -fno-omit-frame-pointer -Wl,-z,relro,-z,now -Wl,-z,noexecstack -fuse-ld=bfd -Ldeps/spdk-v22.09/build/lib -Wl,--whole-archive -Wl,--no-as-needed -lspdk_bdev_malloc -lspdk_bdev_null -lspdk_bdev_nvme -lspdk_bdev_passthru -lspdk_bdev_lvol -lspdk_bdev_raid -lspdk_bdev_error -lspdk_bdev_gpt -lspdk_bdev_split -lspdk_bdev_delay -lspdk_bdev_zone_block -lspdk_blobfs_bdev -lspdk_blobfs -lspdk_blob_bdev -lspdk_lvol -lspdk_blob -lspdk_nvme -lspdk_bdev_ftl -lspdk_ftl -lspdk_bdev_aio -lspdk_bdev_virtio -lspdk_virtio -lspdk_v+ c++ -pthread -g -O3 -march=native -Wall -Wextra -Werror -Wno-subobject-linkage tools/tree_r.cpp -o bin/tree_r_cpu -Iposeidon -Ideps/sppark -Ideps/blst/src -L deps/blst -lblst
fio_user -lspdk_accel_ioat -lspdk_ioat -lspdk_scheduler_dynamic -lspdk_env_dpdk -lspdk_scheduler_dpdk_governor -lspdk_scheduler_gscheduler -lspdk_sock_posix -lspdk_event -lspdk_event_bdev -lspdk_bdev -lspdk_notify -lspdk_dma -lspdk_event_accel -lspdk_accel -lspdk_event_vmd -lspdk_vmd -lspdk_event_sock -lspdk_init -lspdk_thread -lspdk_trace -lspdk_sock -lspdk_rpc -lspdk_jsonrpc -lspdk_json -lspdk_util -lspdk_log -Wl,--no-whole-archive deps/spdk-v22.09/build/lib/libspdk_env_dpdk.a -Wl,--whole-archive deps/spdk-v22.09/dpdk/build/lib/librte_bus_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_cryptodev.a deps/spdk-v22.09/dpdk/build/lib/librte_dmadev.a deps/spdk-v22.09/dpdk/build/lib/librte_eal.a deps/spdk-v22.09/dpdk/build/lib/librte_ethdev.a deps/spdk-v22.09/dpdk/build/lib/librte_hash.a deps/spdk-v22.09/dpdk/build/lib/librte_kvargs.a deps/spdk-v22.09/dpdk/build/lib/librte_mbuf.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_net.a deps/spdk-v22.09/dpdk/build/lib/librte_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_power.a deps/spdk-v22.09/dpdk/build/lib/librte_rcu.a deps/spdk-v22.09/dpdk/build/lib/librte_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_telemetry.a deps/spdk-v22.09/dpdk/build/lib/librte_vhost.a -Wl,--no-whole-archive -lnuma -ldl -Ldeps/spdk-v22.09/isa-l/.libs -lisal -pthread -lrt -luuid -lssl -lcrypto -lm -laio -Ldeps/blst -lblst -L/usr/local/cuda-12.2/lib64 -lcudart_static -lgmp -lconfig++
wait
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -g -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -Xcompiler -march=native -O3 -x cu tools/tree_r.cpp -o bin/tree_r -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -g -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -Xcompiler -march=native -O3 -x cu tools/pc2.cu -o bin/pc2 -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -g -O3 -Xcompiler -march=native -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -x cu tools/tree_r.cpp -o bin/tree_r -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
SECTOR_SIZE_FEATURE=
'[' 32GiB == 512MiB ']'
cd demos/rust
env 'RUSTFLAGS=-C target-cpu=native' cargo +nightly build --release
Compiling supra-seal-demo v0.1.0 (/home/worker/supra_seal/demos/rust)
Finished release [optimized] target(s) in 4.16s
sudo ./target/release/supra-seal-demo
Initializing spdk using config supra_seal.cfg
sectors 2
coord 8 hashers 1
sectors 4
coord 8 hashers 2
sectors 8
coord 8 hashers 4
sectors 16
coord 8 hashers 8
sectors 32
coord 8 hashers 14
coord 16 hashers 2
sectors 64
coord 8 hashers 14
coord 16 hashers 14
coord 24 hashers 4
sectors 128
coord 8 hashers 14
coord 16 hashers 14
coord 24 hashers 14
coord 32 hashers 14
coord 40 hashers 8
TELEMETRY: No legacy callbacks, legacy socket not created
Attaching to 0000:01:00.0
Sealing Parameters
SECTOR_SIZE 34359738368
NODE_SIZE 32
NODE_WORDS 8
NODE_COUNT 1073741824
PARENT_COUNT_BASE 6
PARENT_COUNT_EXP 8
PARENT_COUNT 14
PARENT_SIZE 4
LAYER_COUNT 11
NODES_PER_HASHER 2
PARENT_BUFFER_NODES 3407872
NODE_BUFFER_NODES 524288

Max Offset returned 6fc7d256
Slot size returned 2c0000000 for 128 sectors
max_offset 1875366486 and slot_size 11811160064
**** 2023-07-24 15:35:48 1690184148 Batch 0 locked slot 0
**** 2023-07-24 15:35:48 1690184148 Batch 0 start PC1
supra-seal-demo: pc1/../nvme/spdk_ptr_t.hpp:29: void spdk_ptr_t::alloc(size_t) [with T = batch_t<page_t<sealing_config_t<128> >, 13>; size_t = long unsigned int]: Assertion `ptr != nullptr' failed.
./exec.sh: line 22: 53222 Aborted sudo ./target/release/supra-seal-demo

How do i fix this error？

OS:Ubuntu 20.04.3 LTS

GCC:gcc version 11.4.0 (Ubuntu 11.4.0-2ubuntu1~20.04)

Answer 1 · 2023-07-24T17:19:36.000Z

It looks like spdk failed to allocate storage for PC1. Can you check for the proper allocation of huge pages? How much memory is in the system overall?

grep Huge /proc/meminfo

Look for:

HugePages_Total: 128
HugePages_Free: 128

Answer 2 · 2023-07-25T02:07:29.000Z

I have already set up huge page

:~/supra_seal$ grep Huge /proc/meminfo
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
FileHugePages: 0 kB
HugePages_Total: 128
HugePages_Free: 128
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
Hugetlb: 262144 kB

The system has a total of 1T of memory

Answer 3 · 2023-07-25T05:54:41.000Z

sudo ./scripts/setup.sh status
Hugepages
node hugesize free / total
node0 1048576kB 0 / 0
node0 2048kB 128 / 128

Type BDF Vendor Device NUMA Driver Device Block devices
NVMe 0000:01:00.0 1b96 2400 0 uio_pci_generic - -
NVMe 0000:02:00.0 1b96 2400 0 nvme nvme1 nvme1n1
NVMe 0000:03:00.0 144d a808 0 nvme nvme2 nvme2n1

Is it because there are two node0?

Answer 4 · 2023-07-25T16:33:49.000Z

It could be. If you end up on the node with no pages allocated it seems like that would be a problem. Can you allocate pages on both nodes, or direct it to run on the node with pages allocated?

Answer 5 · 2023-07-26T05:17:57.000Z

Try it later, thank you

Answer 6 · 2023-08-18T13:20:15.000Z

If problem persists, just re-open the issue. Cheers.