x86 Thin LTO + allyesconfig causes `ld.lld: error: kernel image bigger than KERNEL_IMAGE_SIZE`
emojifreak opened this issue · 3 comments
emojifreak commented
This might be a duplicate of #1424 (comment). Thin LTO + allyesconfig gives the following build error. It does not happen with LTO_NON=y. In addition, tools/objtool needs 41 GiB of virtual memory and an extra swap space was needed.
+ printf %-7s %s\n LD .tmp_vmlinux.kallsyms1
LD .tmp_vmlinux.kallsyms1
+ shift
+ [ -n y ]
+ objs=vmlinux.o
+ libs=
+ [ x86 = um ]
+ wl=
+ ld=ld.lld
+ ldflags=-m elf_x86_64 --thinlto-cache-dir=.thinlto-cache -mllvm -import-instr-limit=5 --emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=warn
+ ldlibs=
+ ldflags=-m elf_x86_64 --thinlto-cache-dir=.thinlto-cache -mllvm -import-instr-limit=5 --emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=warn --script=./arch/x86/kernel/vmlinux.lds
+ [ .tmp_vmlinux.kallsyms1 != 1 ]
+ ldflags=-m elf_x86_64 --thinlto-cache-dir=.thinlto-cache -mllvm -import-instr-limit=5 --emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=warn --script=./arch/x86/kernel/vmlinux.lds --strip-debug
+ [ -n y ]
+ ldflags=-m elf_x86_64 --thinlto-cache-dir=.thinlto-cache -mllvm -import-instr-limit=5 --emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=warn --script=./arch/x86/kernel/vmlinux.lds --strip-debug -Map=.tmp_vmlinux.kallsyms1.map
+ ld.lld -m elf_x86_64 --thinlto-cache-dir=.thinlto-cache -mllvm -import-instr-limit=5 --emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=warn --script=./arch/x86/kernel/vmlinux.lds --strip-debug -Map=.tmp_vmlinux.kallsyms1.map -o .tmp_vmlinux.kallsyms1 --whole-archive vmlinux.o --no-whole-archive --start-group --end-group .btf.vmlinux.bin.o
ld.lld: error: kernel image bigger than KERNEL_IMAGE_SIZE
ld.lld: error: kernel image bigger than KERNEL_IMAGE_SIZE
ld.lld: error: kernel image bigger than KERNEL_IMAGE_SIZE
make: *** [Makefile:1161: vmlinux] Error 1
make: Target 'all' not remade because of errors.
$ LANG=C ls -l .*vmlin* *vmlinu*
-rwxr-xr-x 1 ryutaroh ryutaroh 1126946928 Dec 25 20:28 .btf.vmlinux.bin.o
-rwxr-xr-x 1 ryutaroh ryutaroh 8925382200 Dec 25 20:28 .tmp_vmlinux.btf
-rw-r--r-- 1 ryutaroh ryutaroh 255177557 Dec 25 16:57 .tmp_vmlinux.btf.map
lrwxrwxrwx 1 ryutaroh ryutaroh 60 Dec 25 11:15 vmlinux-gdb.py -> /var/tmp/tmp31/x86/linux-5.16-rc6/scripts/gdb/vmlinux-gdb.py
-rw-r--r-- 1 ryutaroh ryutaroh 8378741416 Dec 25 16:57 vmlinux.o
-rw-r--r-- 1 ryutaroh ryutaroh 1546109 Dec 25 16:57 vmlinux.symvers
A reproducing script is below:
#!/bin/sh
KVER=5.16-rc6
LANG=C.UTF-8
export LANG
cd /var/tmp/tmp31
#wget -T 10 -c https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-${KVER}.tar.xz
for a in x86; do
rm -rf /var/tmp/tmp31/$a
mkdir /var/tmp/tmp31/$a
cd /var/tmp/tmp31/$a
exec </dev/null >build-log-${KVER}-${a}.txt 2>&1
set -xe
tar zxf /var/tmp/linux-${KVER}.tar.gz
cd linux-${KVER}
# sed -i 's/-O2/-O3/g' Makefile
# sed -i 's/-Os/-Oz/g' Makefile
make LLVM=1 LLVM_IAS=1 allyesconfig
cat >>.config <<EOF
CONFIG_RUNTIME_TESTING_MENU=n
CONFIG_WERROR=n
CONFIG_XFS_FS=n
CONFIG_OVERLAY_FS=n
CONFIG_EMBEDDED=n
CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT=n
CONFIG_KASAN=n
CONFIG_GCOV_KERNEL=n
CONFIG_COMPILE_TEST=n
CONFIG_TRIM_UNUSED_KSYMS=n
CONFIG_INIT_STACK_ALL_ZERO=y
CONFIG_KVM=y
CONFIG_PREEMPT=y
CONFIG_PREEMPT_DYNAMIC=y
CONFIG_LTO_CLANG_THIN=y
CONFIG_LTO_CLANG_NONE=n
CONFIG_CFI_CLANG=y
CONFIG_CFI_CLANG_SHADOW=y
CONFIG_SHADOW_CALL_STACK=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF5=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_GDB_SCRIPTS=y
CONFIG_MATOM=y
CONFIG_X86_INTEL_TSX_MODE_ON=y
CONFIG_USB_G_DBGP_PRINTK=y
CONFIG_ROMFS_BACKED_BY_BOTH=y
CONFIG_X86_DECODER_SELFTEST=y
CONFIG_UNWINDER_FRAME_POINTER=y
EOF
yes '' |
make --keep-going -j 6 ARCH=$a LLVM=1 LLVM_IAS=1 KCFLAGS="-mllvm -polly-ast-use-context -mllvm -polly-invariant-load-hoisting -mllvm -polly-opt-fusion=max -mllvm -polly-run-inliner -mllvm -polly-vectorizer=stripmine -mllvm -polly-run-dce" oldconfig
sed -i 's/=m$/=y/g' .config
yes '' |
make --keep-going -j 6 V=1 ARCH=$a LLVM=1 LLVM_IAS=1 KCFLAGS="-mllvm -polly-ast-use-context -mllvm -polly-invariant-load-hoisting -mllvm -polly-opt-fusion=max -mllvm -polly-run-inliner -mllvm -polly-vectorizer=stripmine -mllvm -polly-run-dce" all
exec </dev/null >/dev/null 2>&1
done
wait
emojifreak commented
With LTO_NONE=y, files sizes are much smaller as below:
$ LANG=C ls -l .*vmlin* *vmlinu*
-rwxr-xr-x 1 ryutaroh ryutaroh 318122416 Dec 25 23:27 .btf.vmlinux.bin.o
-rwxr-xr-x 1 ryutaroh ryutaroh 5056530336 Dec 25 23:27 .tmp_vmlinux.btf
-rw-r--r-- 1 ryutaroh ryutaroh 155329010 Dec 25 23:25 .tmp_vmlinux.btf.map
-rwxr-xr-x 1 ryutaroh ryutaroh 627375480 Dec 25 23:27 .tmp_vmlinux.kallsyms1
-rw-r--r-- 1 ryutaroh ryutaroh 116152114 Dec 25 23:28 .tmp_vmlinux.kallsyms1.S
-rw-r--r-- 1 ryutaroh ryutaroh 135618449 Dec 25 23:27 .tmp_vmlinux.kallsyms1.map
-rw-r--r-- 1 ryutaroh ryutaroh 19366048 Dec 25 23:28 .tmp_vmlinux.kallsyms1.o
-rwxr-xr-x 1 ryutaroh ryutaroh 646249872 Dec 25 23:28 .tmp_vmlinux.kallsyms2
-rw-r--r-- 1 ryutaroh ryutaroh 116152114 Dec 25 23:28 .tmp_vmlinux.kallsyms2.S
-rw-r--r-- 1 ryutaroh ryutaroh 135619315 Dec 25 23:28 .tmp_vmlinux.kallsyms2.map
-rw-r--r-- 1 ryutaroh ryutaroh 19366048 Dec 25 23:28 .tmp_vmlinux.kallsyms2.o
-rw-r--r-- 1 ryutaroh ryutaroh 1177 Dec 25 23:29 .vmlinux.cmd
-rwxr-xr-x 1 ryutaroh ryutaroh 5113886672 Dec 25 23:29 vmlinux
lrwxrwxrwx 1 ryutaroh ryutaroh 60 Dec 25 21:46 vmlinux-gdb.py -> /var/tmp/tmp32/x86/linux-5.16-rc6/scripts/gdb/vmlinux-gdb.py
-rw-r--r-- 1 ryutaroh ryutaroh 155329959 Dec 25 23:28 vmlinux.map
-rw-r--r-- 1 ryutaroh ryutaroh 5055556384 Dec 25 23:24 vmlinux.o
-rw-r--r-- 1 ryutaroh ryutaroh 1546109 Dec 25 23:24 vmlinux.symvers
nathanchance commented
This comes from arch/x86/kernel/vmlinux.lds.S:
/*
* The ASSERT() sink to . is intentional, for binutils 2.14 compatibility:
*/
. = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE),
"kernel image bigger than KERNEL_IMAGE_SIZE");
I am sure the objtool memory usage is just a symptom of the fact that the kernel is super large, as objtool has to run on vmlinux.o since that is an ELF file, rather than LLVM bitcode.
I am just curious, does this happen without CONFIG_DEBUG_INFO? That option is going to significantly increase the size of the binary. I see about around a five times increase (without LTO, since I don't currently have the time to allow two ThinLTO allyesconfig kernels build):
$ git diff --no-index .build/x86_64-allyesconfig{,-debug}/.config
diff --git a/.build/x86_64-allyesconfig/.config b/.build/x86_64-allyesconfig-debug/.config
index 5b0c10f57363..92162806b0d6 100644
--- a/.build/x86_64-allyesconfig/.config
+++ b/.build/x86_64-allyesconfig-debug/.config
@@ -14548,10 +14548,16 @@ CONFIG_DEBUG_MISC=y
#
# Compile-time checks and compiler options
#
-CONFIG_DEBUG_INFO_NONE=y
+CONFIG_DEBUG_INFO=y
+# CONFIG_DEBUG_INFO_NONE is not set
# CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT is not set
# CONFIG_DEBUG_INFO_DWARF4 is not set
-# CONFIG_DEBUG_INFO_DWARF5 is not set
+CONFIG_DEBUG_INFO_DWARF5=y
+# CONFIG_DEBUG_INFO_REDUCED is not set
+# CONFIG_DEBUG_INFO_COMPRESSED is not set
+# CONFIG_DEBUG_INFO_SPLIT is not set
+CONFIG_PAHOLE_HAS_SPLIT_BTF=y
+# CONFIG_GDB_SCRIPTS is not set
CONFIG_FRAME_WARN=2048
CONFIG_STRIP_ASM_SYMS=y
CONFIG_HEADERS_INSTALL=y
$ diskus .build/x86_64-allyesconfig/vmlinux
1.34 GB (1,338,589,184 bytes)
$ diskus .build/x86_64-allyesconfig-debug/vmlinux
5.75 GB (5,748,252,672 bytes)emojifreak commented
I am just curious, does this happen without CONFIG_DEBUG_INFO? That option is going to significantly increase the size of the binary.
You are absolutely right. With DEBUG_INFO=n, the size of vmlinux.o becomes 1/5, and this symptom disappears.