[BUG]`Frida CRITICAL: Unable to locate the libc` When Attaching to ClickHouse Binaries
fr0m-scratch opened this issue · 4 comments
Describe the bug
Frida is unable to locate the libc
To Reproduce
git clone https://github.com/ClickHouse/ClickHouseand come clickhouse from source codebpftime load ./bpf_progPlease see my source file below- Run the following command to start clickhouse with bpftime
bpftime start /mnt/fast25/ClickHouse/build/programs/clickhouse
or
LD_PRELOAD=/lib/x86_64-linux-gnu/libc.so.6 LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu bpftime start /mnt/fast25/ClickHouse/build/programs/clickhouse
or
LD_PRELOAD=/lib/x86_64-linux-gnu/libc.so.6 LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu LD_P RELOAD=~/.bpftime/libbpftime-agent.so /mnt/fast25/ClickHouse/build/programs/clickhouse
Expected behavior
Attach all Uprobes successfully, as logical identical bpf programs have been attached to other binaries such as rocksdb's db_bench.
Screenshots
Showing Error [Frida CRITICAL] Unable to locate the libc; please file a bug
Desktop (please complete the following information):
- Kernel: Linux 6.2.0-36-generic
- Arch: x86_64
** Related Information
Binaries:
Linked Libraries:
Additional context
My userspace code:
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
#include "attach_override.h"
#include "clickhouse-uring.skel.h"
#include <argp.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <time.h>
#include <unistd.h>
static volatile sig_atomic_t exiting = 0;
static int libbpf_print_fn(enum libbpf_print_level level, const char *format,
va_list args) {
return vfprintf(stderr, format, args);
}
static void sig_int(int signo) { exiting = 1; }
int main(int argc, char **argv) {
LIBBPF_OPTS(bpf_object_open_opts, open_opts);
struct clickhouse_uring_bpf *obj;
int err;
libbpf_set_print(libbpf_print_fn);
obj = clickhouse_uring_bpf__open_opts(&open_opts);
if (!obj) {
fprintf(stderr, "Failed to open BPF object\n");
return 1;
}
err = clickhouse_uring_bpf__load(obj);
if (err) {
fprintf(stderr, "Failed to load BPF object: %d\n", err);
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_start_patch),
"/mnt/fast25/ClickHouse/build/programs/clickhouse", "main");
if (err) {
fprintf(stderr, "Failed to attach BPF program to open\n");
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_write_patch),
"/lib/x86_64-linux-gnu/libc.so.6", "write");
if (err) {
fprintf(stderr, "Failed to attach BPF program to write\n");
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_read_patch),
"/lib/x86_64-linux-gnu/libc.so.6", "read");
if (err) {
fprintf(stderr, "Failed to attach BPF program to read\n");
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_pread_patch),
"/lib/x86_64-linux-gnu/libc.so.6", "pread");
if (err) {
fprintf(stderr, "Failed to attach BPF program to pread\n");
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_send_patch),
"/lib/x86_64-linux-gnu/libc.so.6", "send");
if (err) {
fprintf(stderr, "Failed to attach BPF program to send\n");
goto cleanup;
}
err = bpf_prog_attach_uprobe_with_override(
bpf_program__fd(obj->progs.bpf_recv_patch),
"/lib/x86_64-linux-gnu/libc.so.6", "recv");
if (err) {
fprintf(stderr, "Failed to attach BPF program to recv\n");
goto cleanup;
}
if (signal(SIGINT, sig_int) == SIG_ERR) {
fprintf(stderr, "Can't set signal handler: %s\n", strerror(errno));
err = 1;
goto cleanup;
}
while (!exiting) {
// Main polling loop
}
cleanup:
clickhouse_uring_bpf__destroy(obj);
return err != 0;
}
MakeFile that adapted from the example folder:
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
OUTPUT := .output
CLANG ?= clang
LIBBPF_SRC := $(abspath /mnt/fast25/bpftime/third_party/libbpf/src)
BPFTOOL_SRC := $(abspath /mnt/fast25/bpftime/third_party/bpftool/src)
LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
| sed 's/arm.*/arm/' \
| sed 's/aarch64/arm64/' \
| sed 's/ppc64le/powerpc/' \
| sed 's/mips.*/mips/' \
| sed 's/riscv64/riscv/' \
| sed 's/loongarch64/loongarch/')
VMLINUX := /mnt/fast25/bpftime/third_party/vmlinux/$(ARCH)/vmlinux.h
# Use our own libbpf API headers and Linux UAPI headers distributed with
# libbpf to avoid dependency on system-wide headers, which could be missing or
# outdated
INCLUDES := -I$(OUTPUT) -I/mnt/fast25/bpftime/third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
CFLAGS := -g -Wall
ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
APPS = clickhouse-uring# minimal minimal_legacy uprobe kprobe fentry usdt sockfilter tc ksyscall
CARGO ?= $(shell which cargo)
ifeq ($(strip $(CARGO)),)
BZS_APPS :=
else
BZS_APPS := # profile
APPS += $(BZS_APPS)
# Required by libblazesym
ALL_LDFLAGS += -lrt -ldl -lpthread -lm
endif
# Get Clang's default includes on this system. We'll explicitly add these dirs
# to the includes list when compiling with `-target bpf` because otherwise some
# architecture-specific dirs will be "missing" on some architectures/distros -
# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
# sys/cdefs.h etc. might be missing.
#
# Use '-idirafter': Don't interfere with include mechanics except where the
# build would have failed anyways.
CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
ifeq ($(V),1)
Q =
msg =
else
Q = @
msg = @printf ' %-8s %s%s\n' \
"$(1)" \
"$(patsubst $(abspath $(OUTPUT))/%,%,$(2))" \
"$(if $(3), $(3))";
MAKEFLAGS += --no-print-directory
endif
define allow-override
$(if $(or $(findstring environment,$(origin $(1))),\
$(findstring command line,$(origin $(1)))),,\
$(eval $(1) = $(2)))
endef
$(call allow-override,CC,$(CROSS_COMPILE)cc)
$(call allow-override,LD,$(CROSS_COMPILE)ld)
.PHONY: all
all: $(APPS)
.PHONY: clean
clean:
$(call msg,CLEAN)
$(Q)rm -rf $(OUTPUT) $(APPS)
$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $@
# Build libbpf
$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
$(call msg,LIB,$@)
$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1 \
OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@) \
INCLUDEDIR= LIBDIR= UAPIDIR= \
install
# Build bpftool
$(BPFTOOL): | $(BPFTOOL_OUTPUT)
$(call msg,BPFTOOL,$@)
$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB, $@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
$(call msg,LIB,$@)
$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
# Build BPF code
$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
$(call msg,BPF,$@)
$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \
$(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) \
-c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
# Generate BPF skeletons
$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
$(call msg,GEN-SKEL,$@)
$(Q)$(BPFTOOL) gen skeleton $< > $@
# Build user-space code
$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
$(call msg,CC,$@)
$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
$(BZS_APPS): $(LIBBLAZESYM_OBJ)
# Build application binary
$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
$(call msg,BINARY,$@)
$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
# delete failed targets
.DELETE_ON_ERROR:
# keep intermediate (.skel.h, .bpf.o, etc) targets
.SECONDARY:
bench: bench.cpp
$(CXX) bench.cpp -Wall -g -o bench
Have you tried using bpftime attach instead of bpftime start?
Also, could you please send me the file /mnt/fast25/ClickHouse/build/programs/clickhouse so I can investigate it? Thank you!
Thank you for the reply! bpftime attach produces similar problem with one more line of Aborted (core dumpted) as shown below
Below is the link to my clickhouse binary for Linux 6.2.0-36-generic on x86_64:
https://drive.google.com/file/d/1KZ6I6vGFJyrVA0aztfRTHGUFTUS--vsi/view?usp=sharing
You can also git clone https://github.com/ClickHouse/ClickHouse and compile from source, since no changes have been made to ClickHouse yet. Thank you very much for your help, and please let me know if any info is needed.
After a bunch of debugging with gdb, I found the root cause:
In short, frida will use dlopen to get the dynamic section address of libc, at https://github.com/frida/frida-gum/blob/04bc42f0c6c99cc350f2b0d38bd2d1db2032bc88/gum/backend-linux/gumprocess-linux.c#L1955. If call to this dlopen fails, frida another way to detect libc (at https://github.com/frida/frida-gum/blob/04bc42f0c6c99cc350f2b0d38bd2d1db2032bc88/gum/backend-linux/gumprocess-linux.c#L2003), but frida will not treat /lib/x86_64-linux-gnu/libc.so.6 (File name appeared in ELF, is a symlink) the same file as /usr/lib/x86_64-linux-gnu/libc.so.6(The actual file), (See https://github.com/frida/frida-gum/blob/04bc42f0c6c99cc350f2b0d38bd2d1db2032bc88/gum/backend-linux/gumprocess-linux.c#L2013 for how frida match paths) so frida won't be able to detect libc
The solution can be simply described: patch clickhouse to allow dynamic library loading
Thank you so much for your extensive investigation! I will try to patch ClickHouse with dynamic library loading.