loading data from dump files, bulk insert to another es instance,fatal error: runtime: out of memory
np7sky opened this issue · 1 comments
dump elasticsearch documents into local file
./esm -s http://localhost:9200 -x "console*" -t "1m" -c 10 --buffer_count=10 --refresh -o old-console.json
[04-01 14:59:21] [INF] [main.go:474,main] start data migration..
Scroll 3709022 / 3709022 [===========================================================================================================================================================================] 100.00% 11m4s
Write 3709022 / 3709022 [============================================================================================================================================================================] 100.00% 11m4s
[04-01 15:10:25] [INF] [main.go:505,main] data migration finished.
old-console.json size is 20g.
execute：
esm -d https://192.168.0.12:9200 -y "old-console" -n username:password -t "1m" --sliced_scroll_size=10 -c 10 --buffer_count=10 -b 5 --refresh -i old-console.json
expections：
Scroll 0 / 1 [---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------] 0.00%
Bulk 160643 / 523953 [=====================================================>------------------------------------------------------------------------------------------------------------------------] 30.66% 01m41s
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xa120b8, 0x16)
/usr/local/go/src/runtime/panic.go:1116 +0x72
runtime.sysMap(0xc190000000, 0x108000000, 0xe4e3b8)
/usr/local/go/src/runtime/mem_linux.go:169 +0xc6
runtime.(*mheap).sysAlloc(0xe336a0, 0x107800000, 0x42bd97, 0xe336a8)
/usr/local/go/src/runtime/malloc.go:727 +0x1e5
runtime.(*mheap).grow(0xe336a0, 0x83aa9, 0x0)
/usr/local/go/src/runtime/mheap.go:1344 +0x85
runtime.(*mheap).allocSpan(0xe336a0, 0x83aa9, 0x457dd9ad0100, 0xe4e3c8, 0xfffffffffffffade)
/usr/local/go/src/runtime/mheap.go:1160 +0x6b6
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:907 +0x65
runtime.(*mheap).alloc(0xe336a0, 0x83aa9, 0xc000280101, 0xc000000a80)
/usr/local/go/src/runtime/mheap.go:901 +0x85
runtime.largeAlloc(0x107551d45, 0xe10101, 0xc00003df01)
/usr/local/go/src/runtime/malloc.go:1177 +0x92
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:1071 +0x46
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:370 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1116
goroutine 36 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc0003d9aa0 sp=0xc0003d9a98 pc=0x46d700
runtime.mallocgc(0x107551d45, 0x954340, 0x518c01, 0xa35d90)
/usr/local/go/src/runtime/malloc.go:1070 +0x938 fp=0xc0003d9b40 sp=0xc0003d9aa0 pc=0x40ddf8
runtime.makeslice(0x954340, 0x107551d45, 0x107551d45, 0xc0003d9bc0)
/usr/local/go/src/runtime/slice.go:98 +0x6c fp=0xc0003d9b70 sp=0xc0003d9b40 pc=0x44e98c
bytes.makeSlice(0x107551d45, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73 fp=0xc0003d9bb0 sp=0xc0003d9b70 pc=0x483db3
bytes.(*Buffer).grow(0xc00006be90, 0xa59ec5, 0xc0003d9c50)
/usr/local/go/src/bytes/buffer.go:142 +0x156 fp=0xc0003d9c00 sp=0xc0003d9bb0 pc=0x4836d6
bytes.(*Buffer).Write(0xc00006be90, 0xc166704000, 0xa59ec5, 0xaee654, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:172 +0xd0 fp=0xc0003d9c30 sp=0xc0003d9c00 pc=0x4839d0
encoding/json.(*Encoder).Encode(0xc0003d9ee8, 0x975ec0, 0xc000548450, 0x0, 0x0)
/usr/local/go/src/encoding/json/stream.go:231 +0x1cb fp=0xc0003d9cc0 sp=0xc0003d9c30 pc=0x516a8b
main.(*Migrator).NewBulkWorker(0xc000196000, 0xc000432bb8, 0xc0001aa160, 0xc00018c3e0)
/Users/medcl/go/src/infini.sh/esm/bulk.go:126 +0xc13 fp=0xc0003d9fc0 sp=0xc0003d9cc0 pc=0x8f9333
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc0003d9fc8 sp=0xc0003d9fc0 pc=0x46f341
created by main.main
/Users/medcl/go/src/infini.sh/esm/main.go:483 +0x65d
goroutine 1 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc00018c3e8)
/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc00018c3e0)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
main.main()
/Users/medcl/go/src/infini.sh/esm/main.go:492 +0x488
goroutine 6 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc0001066d0, 0x0)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0001066c0)
/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cihub/seelog.(*asyncLoopLogger).processItem(0xc000142000, 0x0)
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:50 +0x8a
github.com/cihub/seelog.(*asyncLoopLogger).processQueue(0xc000142000)
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:63 +0x45
created by github.com/cihub/seelog.NewAsyncLoopLogger
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:40 +0x9c
goroutine 7 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc000106850, 0x0)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000106840)
/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cihub/seelog.(*asyncLoopLogger).processItem(0xc000142120, 0x0)
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:50 +0x8a
github.com/cihub/seelog.(*asyncLoopLogger).processQueue(0xc000142120)
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:63 +0x45
created by github.com/cihub/seelog.NewAsyncLoopLogger
/Users/medcl/go/src/github.com/cihub/seelog/behavior_asynclooplogger.go:40 +0x9c
goroutine 11 [chan send]:
main.(*Migrator).NewFileReadWorker(0xc000196000, 0xc0001aa000, 0xc00018c3e0)
/Users/medcl/go/src/infini.sh/esm/file.go:61 +0x38c
created by main.main
/Users/medcl/go/src/infini.sh/esm/main.go:235 +0x378a
goroutine 18 [select, 1 minutes]:
github.com/cheggaaa/pb.catchTerminate(0xc0000680c0)
/Users/medcl/go/src/github.com/cheggaaa/pb/pb_x.go:112 +0x18c
created by github.com/cheggaaa/pb.lockEcho
/Users/medcl/go/src/github.com/cheggaaa/pb/pb_x.go:83 +0x398
goroutine 19 [select]:
github.com/cheggaaa/pb.(*Pool).writer(0xc00005cfc0)
/Users/medcl/go/src/github.com/cheggaaa/pb/pool.go:77 +0x115
created by github.com/cheggaaa/pb.(*Pool).Start
/Users/medcl/go/src/github.com/cheggaaa/pb/pool.go:60 +0xc5
goroutine 50 [syscall, 1 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:147 +0x9d
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:150 +0x45
goroutine 24 [sleep]:
time.Sleep(0x3b9aca00)
/usr/local/go/src/runtime/time.go:188 +0xbf
infini.sh/framework/lib/fasthttp.(*TCPDialer).tcpAddrsClean(0xe0cba0)
/Users/medcl/go/src/infini.sh/framework/lib/fasthttp/tcpdialer.go:391 +0x125
created by infini.sh/framework/lib/fasthttp.(*TCPDialer).dial.func1
/Users/medcl/go/src/infini.sh/framework/lib/fasthttp/tcpdialer.go:276 +0x6a
use --buffer_count=100000 to control memory buffer size.