100 eventlisteners, supervisord_golang can not start
FYPYTHON opened this issue · 2 comments
100 eventlisteners, supervisord_golang can not start
machine info：
Linux a6506b46-e31b-11ed-9225-000c29af87ae 4.19.90-2106.3.0.0095.oe1.x86_64 #1 SMP Wed Jun 23 15:18:59 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
NAME="openEuler"
VERSION="20.03 (LTS-SP2)"
ID="openEuler"
VERSION_ID="20.03"
PRETTY_NAME="openEuler 20.03 (LTS-SP2)"
ANSI_COLOR="0;31"
Mem: 32G
processor: total 8
event.sh
[eventlistener:event_test0]
command = event.sh
events = PROCESS_STATE_RUNNING,TICK_60
priority = 1000
autostart = true
autorestart = true
stdout_logfile = /supervisord/event.log
buffer_size = 30
stderr_logfile = /supervisord/event.log
event_test0-event_test100
`
fatal error: concurrent map writes
fatal error: concurrent map writes
fatal error: concurrent map writes
goroutine 132 [running]:
github.com/ochinchina/supervisord/events.(*EventListenerManager).registerEventListener(0xc000062dc0, {0xc0004b40ae, 0xc}, {0xc0000399e0, 0x2, 0xc000216440?}, 0xc00007eb90)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/events/events.go:317 +0x77
github.com/ochinchina/supervisord/events.RegisterEventListener(...)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/events/events.go:345
github.com/ochinchina/supervisord/process.(*Process).registerEventListener(0xc0000ff3f0, {0xc0004b40ae, 0xc}, {0xc0000399e0, 0x2, 0x2}, {0xab2660, 0xc000216440}, {0xab26a0, 0xc000216458})
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:808 +0xda
github.com/ochinchina/supervisord/process.(*Process).setLog(0xc0000ff3f0)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:774 +0xe9d
github.com/ochinchina/supervisord/process.(*Process).createProgramCommand(0xc0000ff3f0)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:422 +0x17f
github.com/ochinchina/supervisord/process.(*Process).run(0xc0000ff3f0, 0xc00037ea80)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:558 +0x525
github.com/ochinchina/supervisord/process.(*Process).Start.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:162 +0x92
created by github.com/ochinchina/supervisord/process.(*Process).Start
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:159 +0x385
goroutine 1 [sleep]:
time.Sleep(0x3b9aca00)
/opt/midware/go/src/runtime/time.go:195 +0x135
main.(*Supervisor).startEventListeners(0xc000218a80)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/supervisor.go:505 +0x9d
main.(*Supervisor).Reload(0xc000218a80)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/supervisor.go:450 +0x26c
main.runServer()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/main.go:132 +0x23d
main.Daemonize({0xc00023a100, 0x35}, 0xa288a8)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/daemonize.go:26 +0x26a
main.main()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/main.go:171 +0x17a
goroutine 6 [chan receive]:
github.com/ochinchina/supervisord/events.startTickTimer.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/events/events.go:288 +0xc5
created by github.com/ochinchina/supervisord/events.startTickTimer
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/events/events.go:284 +0xcd
goroutine 34 [sleep]:
time.Sleep(0x2540be400)
/opt/midware/go/src/runtime/time.go:195 +0x135
github.com/ochinchina/filechangemonitor.(*FileChangeMonitor).start.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/filechangemonitor/file_change_monitor.go:118 +0x45
created by github.com/ochinchina/filechangemonitor.(*FileChangeMonitor).start
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/filechangemonitor/file_change_monitor.go:104 +0x56
goroutine 35 [select]:
github.com/robfig/cron/v3.(*Cron).run(0xc000226000)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/robfig/cron/v3/cron.go:263 +0xab9
created by github.com/robfig/cron/v3.(*Cron).Start
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/robfig/cron/v3/cron.go:222 +0xca
goroutine 19 [chan receive]:
main.initSignals.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/main.go:42 +0x46
created by main.initSignals
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/main.go:41 +0xce
goroutine 38 [syscall]:
os/signal.signal_recv()
/opt/midware/go/src/runtime/sigqueue.go:152 +0x2f
os/signal.loop()
/opt/midware/go/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
/opt/midware/go/src/os/signal/signal.go:151 +0x2a
goroutine 21 [syscall]:
syscall.Syscall6(0x200000003?, 0xc00010ab60?, 0xc000040f00?, 0x482b07?, 0xc000516f60?, 0xc000516fc0?, 0x482b3c?)
/opt/midware/go/src/syscall/syscall_linux.go:91 +0x36
os.(*Process).blockUntilWaitable(0xc0000286f0)
/opt/midware/go/src/os/wait_waitid.go:32 +0x87
os.(*Process).wait(0xc0000286f0)
/opt/midware/go/src/os/exec_unix.go:22 +0x28
os.(*Process).Wait(...)
/opt/midware/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0xc0000e1600)
/opt/midware/go/src/os/exec/exec.go:890 +0x45
github.com/ochinchina/supervisord/process.(*Process).waitForExit(0xc0001640b0, 0xc000000005?)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:486 +0x51
github.com/ochinchina/supervisord/process.(*Process).run(0xc0001640b0, 0xc0000122b8)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:599 +0x917
github.com/ochinchina/supervisord/process.(*Process).Start.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:162 +0x92
created by github.com/ochinchina/supervisord/process.(*Process).Start
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:159 +0x385
goroutine 22 [sync.Mutex.Lock]:
sync.runtime_SemacquireMutex(0xc?, 0x0?, 0xffffffffffffffff?)
/opt/midware/go/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xe5b8e0)
/opt/midware/go/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/opt/midware/go/src/sync/mutex.go:90
sync.(*RWMutex).Lock(0xc00054fb00?)
/opt/midware/go/src/sync/rwmutex.go:147 +0x36
syscall.forkExec({0xc000383680?, 0x10?}, {0xc00053c4d0, 0x1, 0x1}, 0x7fb3e343ff18?)
/opt/midware/go/src/syscall/exec_unix.go:200 +0x2e7
syscall.StartProcess(...)
/opt/midware/go/src/syscall/exec_unix.go:255
os.startProcess({0xc000383680, 0x42}, {0xc00053c4d0, 0x1, 0x1}, 0xc000387380)
/opt/midware/go/src/os/exec_posix.go:54 +0x332
os.StartProcess({0xc000383680, 0x42}, {0xc00053c4d0, 0x1, 0x1}, 0x417910?)
/opt/midware/go/src/os/exec.go:109 +0x5a
os/exec.(*Cmd).Start(0xc0000e1b80)
/opt/midware/go/src/os/exec/exec.go:693 +0x5ee
github.com/ochinchina/supervisord/process.(*Process).run(0xc000164160, 0xc0000125d0)
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:564 +0x53f
github.com/ochinchina/supervisord/process.(*Process).Start.func1()
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:162 +0x92
created by github.com/ochinchina/supervisord/process.(*Process).Start
/opt/midware/supervisord_golang/new_version/supervisord-0.7.3/vendor/github.com/ochinchina/supervisord/process/process.go:159 +0x385
`
5个eventlistener，20个program，启动supervisord_golang有很小概率报上述错误。
用100个eventlistener，20个program，启动supervisord_golang就一直报这个了
eventlistener使用有限制吗？
python版本同样数量的配置，没有这个问题
EventListenerManager用map，fatal error: concurrent map writes这个很难保证一定不出。
在struct EventListenerManager中加了sync.Mutex。只考虑稳定性，可以这么改下。。。