记一次僵尸进程分析过程
Opened this issue · 0 comments
jason--liu commented
背景
服务器上有一个虚机发现删不掉, ps命令查看发现qemu-kvm变成了zombie.
僵尸进程出现的原因, 一般是子进程退出时父进程没有回收子进程的资源, 这个时候子进程就会变成僵尸进程, 或者子进程根本没有发送SIGCHLD通知父进程回收
分析过程
查看进程状态
[root@node-3 ~]# cat /proc/22889/status
Name: qemu-kvm
State: Z (zombie)
Tgid: 22889
Ngid: 22985
Pid: 22889
PPid: 1
TracerPid: 0
Uid: 42436 42436 42436 42436
Gid: 36 36 36 36
FDSize: 0
Groups: 10 36 42400 42427 42436
NStgid: 22889
NSpid: 22889
NSpgid: 22807
NSsid: 22807
...
的确是Z state. 再查看22889此时的父进程是systemd. 查看当前系统systemd运行正常, 相关命令也能正常执行, 不存在”卡住”的情况. 怀疑是不是systemd只回收自己创建的进程, 而不回收托孤进程. 验证了下
#include <stdio.h>
#include <sys/wait.h>
#include <stdlib.h>
#include <unistd.h>
int main(void)
{
pid_t pid,wait_pid;
int status;
pid = fork();
if (pid==-1) {
perror("Cannot create new process");
exit(1);
} else if (pid==0) {
printf("child process id: %ld\n", (long) getpid());
pause();
_exit(0);
} else {
#if 1 /* define 1 to make child process always a zomie */
printf("ppid:%d\n", getpid());
while(1);
#endif
do {
wait_pid=waitpid(pid, &status, WUNTRACED | WCONTINUED);
if (wait_pid == -1) {
perror("cannot using waitpid function");
exit(1);
}
if (WIFEXITED(status))
printf("child process exites, status=%d\n", WEXITSTATUS(status));
if(WIFSIGNALED(status))
printf("child process is killed by signal %d\n", WTERMSIG(status));
if (WIFSTOPPED(status))
printf("child process is stopped by signal %d\n", WSTOPSIG(status));
if (WIFCONTINUED(status))
printf("child process resume running....\n");
} while (!WIFEXITED(status) && !WIFSIGNALED(status));
exit(0);
}
}实验发现, 托孤进程退出时, system正常情况下也会回收.
再查看22889的一些进程状态. 发现22889下面有子线程处于D state
[root@node-3 ~]# cat /proc/22889/task/22976/status
Name: safe_timer
State: D (disk sleep)
Tgid: 22889
Ngid: 0
Pid: 22976
PPid: 1
TracerPid: 6076
Uid: 42436 42436 42436 42436
Gid: 36 36 36 36
FDSize: 0
Groups: 10 36 42400 42427 42436
NStgid: 22889
NSpid: 22976
NSpgid: 22807
NSsid: 22807
Threads: 2
SigQ: 2/260720
SigPnd: 0000000000040000
ShdPnd: 0000000000004100
SigBlk: fffffffe7ffbfeff
SigIgn: 0000000000001000
SigCgt: 0000000180004243
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000
NoNewPrivs: 1
Seccomp: 2
Speculation_Store_Bypass: not vulnerable
Cpus_allowed: ffffffff,fffc0000
Cpus_allowed_list: 18-63
Mems_allowed: 0f
Mems_allowed_list: 0-3
voluntary_ctxt_switches: 11
nonvoluntary_ctxt_switches: 3
查看进程call trace信息
[root@node-3 ~]# cat /proc/22889/task/22976/stack
[<0>] __switch_to+0x6c/0x90
[<0>] exit_aio+0x104/0x120
[<0>] mmput+0x54/0x168
[<0>] vhost_dev_cleanup+0x290/0x2c0 [vhost]
[<0>] vhost_net_release+0x5c/0xd0 [vhost_net]
[<0>] __fput+0x9c/0x1d8
[<0>] ____fput+0x20/0x30
[<0>] task_work_run+0xb0/0xe0
[<0>] do_exit+0x3c0/0xa78
[<0>] do_group_exit+0x3c/0xd0
[<0>] get_signal+0x160/0x810
[<0>] do_notify_resume+0x15c/0x2f8
[<0>] work_pending+0x8/0x10
[<0>] 0xffffffffffffffff
看起来是aio没有返回, 然后schedule出去了.
看起来应该是子线程处于D state, 杀掉主线程时, 主线程就会变成zombie, 需要验证下.
实验验证
在本地实验, 通过fork来模拟托孤进程效果. 子进程中创建线程来读设备文件, 驱动中会讲进程设置为TASK_UNINTERRUPTIBLE(D state)
应用程序:
#include <stdio.h>
#include <pthread.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <linux/unistd.h>
#include <sys/syscall.h>
static pid_t gettid( void )
{
return syscall(__NR_gettid);
}
static void *thread_fun(void *param)
{
int fd = open("/dev/memory", O_RDWR);
printf("fd=%d\n",fd );
char buf[500]={0};
int size = read(fd, buf, 500);
printf("thread pid:%d, tid:%d pthread_self:%lu, fd=%d, size=%d\n", getpid(), gettid(),pthread_self(), fd, size);
return NULL;
}
int main(void)
{
pthread_t tid1, tid2;
pid_t pid;
int ret;
printf("thread pid:%d, tid:%d pthread_self:%lu\n", getpid(), gettid(),pthread_self());
pid = fork();
if (pid==-1) {
perror("Cannot create new process");
exit(1);
} else if (pid==0) {
printf("child process id: %ld\n", (long) getpid());
ret = pthread_create(&tid1, NULL, thread_fun, NULL);
if (ret == -1) {
perror("cannot create new thread"); return -1;
}
pause();
_exit(0);
}
//ret = pthread_create(&tid1, NULL, thread_fun, NULL);
/*
if (pthread_join(tid1, NULL) != 0) {
perror("call pthread_join function fail");
return -1;
}
*/
while(1);
return 0;
}驱动程序部分实现:
// ...
// ...
static DECLARE_WAIT_QUEUE_HEAD(wq);
static volatile int flag = 0;
ssize_t memory_read(struct file *filp, char *buf, size_t count, loff_t *f_pos)
{
printk("<1>going to sleep\n");
flag = 0;
//wait_event_interruptible(wq, flag != 0);
wait_event(wq, flag != 0); //将进程设置为TASK_UNINTERRUPTIBLE
//wait_event_killable(wq, flag != 0);
//pr_info("wake-up by P:0x%x T:0x%x\n", current->signal->shared_pending.signal,current->pending.signal);
printk("<1>Reading from memory module\n");
copy_to_user(buf,memory_buffer,1);
/* Changing reading position as best suits */
if (*f_pos == 0) {
*f_pos+=1;
return 1;
} else {
return 0;
}
}
// ...
// ...编译执行, 父进程pid是2203, 子进程是2204
杀掉父进程, 让子进程2204托孤给systemd.
查看2204子线程状态是处于D state的.
2204这个时候处于S state.
尝试杀掉2204, 再来查看status
可以看到进程变成了zombie 状态.
疑问: 为什么当taskgroup中存在D的task时, taskgroup leader被kill会造成 leader变成zombie?
原因分析
进程被kill时, 会走到do_exit
void __noreturn do_exit(long code)
{
// ...
taskstats_exit(tsk, group_dead);
// 是否各种资源
exit_mm();
if (group_dead)
acct_process();
trace_sched_process_exit(tsk);
exit_sem(tsk);
exit_shm(tsk);
exit_files(tsk);
exit_fs(tsk);
if (group_dead)
disassociate_ctty(1);
exit_task_namespaces(tsk);
exit_task_work(tsk);
exit_thread(tsk);
exit_umh(tsk);
// ...
exit_notify(tsk, group_dead);
}然后走到exit_notify
static void exit_notify(struct task_struct *tsk, int group_dead)
{
bool autoreap;
struct task_struct *p, *n;
LIST_HEAD(dead);
// ...
// 如果是group leader且group不为空, aotoreap=false
else if (thread_group_leader(tsk)) {
autoreap = thread_group_empty(tsk) && // thread_group_empty为false, &&后面的就不会再判断了
do_notify_parent(tsk, tsk->exit_signal);
} else {
autoreap = true;
}
// 将退出状态设置为EXIT_ZOMBIE
tsk->exit_state = autoreap ? EXIT_DEAD : EXIT_ZOMBIE;
if (tsk->do_exitexit_state == EXIT_DEAD)
list_add(&tsk->ptrace_entry, &dead);
//...
list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
list_del_init(&p->ptrace_entry);
release_task(p);
}
}执行完exit_notify之后, 主线程状态设置成了EXIT_ZOMBIE, 成为了zombie, 而且没有通知父进程. 而子线程因为处于D state, 没法返回用户空间, 所以一直不会收到信号.
主线程资源什么时候被回收?
当D task从内核空间返回会检查信号位图, 如果收到信号,会有相关出理操作, 这里kill主线程, D task返回后也会收到SIGKILL, 也会走到do_exit, 然后走到exit_notify, 这一次会进入release_task
void release_task(struct task_struct *p)
{
struct task_struct *leader;
//...
if (leader != p && thread_group_empty(leader)
&& leader->exit_state == EXIT_ZOMBIE) {
/*
* If we were the last child thread and the leader has
* exited already, and the leader's parent ignores SIGCHLD,
* then we are the one who should release the leader.
*/
zap_leader = do_notify_parent(leader, leader->exit_signal);
if (zap_leader)
leader->exit_state = EXIT_DEAD;
}
write_unlock_irq(&tasklist_lock);
release_thread(p);
// 释放task_struct
call_rcu(&p->rcu, delayed_put_task_struct);
// ...
}这里会进入if判断, 然后调用do_notify_parent发送SIGCHLD来通知主线程的parent来"收尸". 子线程然后走到delayed_put_task_struct来释放task_struct.
父进程收到SIGCHLD, 会调用waipid(), 然后又会走到release_task(wait_task_zombie->release_task)来释放主线程的task_struct.
最后,主线程的躯壳也被释放了.