Active Monitor crashing with concurrent map updates
RaviHari opened this issue · 0 comments
RaviHari commented
Describe the bug
Active-Monitor crashin with the following error:
fatal error: concurrent map read and map write
goroutine 5518 [running]:
runtime.throw(0x2245810, 0x21)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc0006e8ff0 sp=0xc0006e8fc0 pc=0x1037c12
runtime.mapaccess2(0x208cf80, 0xc00088ee70, 0xc00082edf0, 0xc00082edf0, 0xc00036f802)
/usr/local/go/src/runtime/map.go:469 +0x25b fp=0xc0006e9030 sp=0xc0006e8ff0 pc=0x101179b
reflect.mapaccess(0x208cf80, 0xc00088ee70, 0xc00082edf0, 0x2238f1d)
/usr/local/go/src/runtime/map.go:1309 +0x3f fp=0xc0006e9068 sp=0xc0006e9030 pc=0x1066aff
reflect.Value.MapIndex(0x208cf80, 0xc00088ee70, 0x15, 0x2037380, 0xc00082edf0, 0x98, 0x21f8c40, 0x208cf80, 0x208cf80)
/usr/local/go/src/reflect/value.go:1188 +0x16e fp=0xc0006e90e0 sp=0xc0006e9068 pc=0x109cbee
encoding/json.mapEncoder.encode(0x22cd1b0, 0xc000650080, 0x208cf80, 0xc00088ee70, 0x15, 0x2080000)
/usr/local/go/src/encoding/json/encode.go:801 +0x30d fp=0xc0006e9258 sp=0xc0006e90e0 pc=0x111896d
encoding/json.mapEncoder.encode-fm(0xc000650080, 0x208cf80, 0xc00088ee70, 0x15, 0x2cb0000)
/usr/local/go/src/encoding/json/encode.go:777 +0x65 fp=0xc0006e9298 sp=0xc0006e9258 pc=0x1124e65
encoding/json.(*encodeState).reflectValue(0xc000650080, 0x208cf80, 0xc00088ee70, 0x15, 0xc0006e0000)
/usr/local/go/src/encoding/json/encode.go:358 +0x82 fp=0xc0006e92d0 sp=0xc0006e9298 pc=0x1115b02
encoding/json.(*encodeState).marshal(0xc000650080, 0x208cf80, 0xc00088ee70, 0x1f50000, 0x0, 0x0)
/usr/local/go/src/encoding/json/encode.go:330 +0xf4 fp=0xc0006e9330 sp=0xc0006e92d0 pc=0x11156f4
encoding/json.(*Encoder).Encode(0xc00061ec80, 0x208cf80, 0xc00088ee70, 0x30, 0x30)
/usr/local/go/src/encoding/json/stream.go:206 +0x8b fp=0xc0006e93c0 sp=0xc0006e9330 pc=0x112294b
go.uber.org/zap/zapcore.(*jsonEncoder).AddReflected(0xc0003d2e10, 0x222b748, 0x8, 0x208cf80, 0xc00088ee70, 0xc0006e94b0, 0x1085ff4)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/json_encoder.go:150 +0x65 fp=0xc0006e9440 sp=0xc0006e93c0 pc=0x1f5f385
go.uber.org/zap/zapcore.Field.AddTo(0x222b748, 0x8, 0x16, 0x0, 0x0, 0x0, 0x208cf80, 0xc00088ee70, 0x240ac20, 0xc0003d2e10)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/field.go:159 +0xb16 fp=0xc0006e9518 sp=0xc0006e9440 pc=0x1f5e3d6
go.uber.org/zap/zapcore.addFields(0x240ac20, 0xc0003d2e10, 0xc00084c800, 0x1, 0x1)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/field.go:199 +0xcf fp=0xc0006e95c0 sp=0xc0006e9518 pc=0x1f5eaaf
go.uber.org/zap/zapcore.consoleEncoder.writeContext(0xc000260000, 0xc00023bc00, 0xc00084c800, 0x1, 0x1)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/console_encoder.go:131 +0xcb fp=0xc0006e9660 sp=0xc0006e95c0 pc=0x1f5a88b
go.uber.org/zap/zapcore.consoleEncoder.EncodeEntry(0xc000260000, 0x0, 0xc027abae3294edc8, 0x5acbf2bbb7f, 0x2cb8ce0, 0xc000041680, 0x17, 0x223ac34, 0x18, 0x0, ...)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/console_encoder.go:110 +0x3df fp=0xc0006e9718 sp=0xc0006e9660 pc=0x1f5a23f
sigs.k8s.io/controller-runtime/pkg/log/zap.(*KubeAwareEncoder).EncodeEntry(0xc000478140, 0x0, 0xc027abae3294edc8, 0x5acbf2bbb7f, 0x2cb8ce0, 0xc000041680, 0x17, 0x223ac34, 0x18, 0x0, ...)
/Users/rhari/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/log/zap/kube_helpers.go:126 +0x175 fp=0xc0006e9930 sp=0xc0006e9718 pc=0x1f78c95
go.uber.org/zap/zapcore.(*ioCore).Write(0xc000260060, 0x0, 0xc027abae3294edc8, 0x5acbf2bbb7f, 0x2cb8ce0, 0xc000041680, 0x17, 0x223ac34, 0x18, 0x0, ...)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/core.go:86 +0xa9 fp=0xc0006e9a08 sp=0xc0006e9930 pc=0x1f5b0c9
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc00016e6e0, 0xc00084c800, 0x1, 0x1)
/Users/rhari/go/pkg/mod/go.uber.org/zap@v1.10.0/zapcore/entry.go:215 +0x12d fp=0xc0006e9ba8 sp=0xc0006e9a08 pc=0x1f5cb4d
github.com/go-logr/zapr.(*infoLogger).Info(0xc0004781c8, 0x223ac34, 0x18, 0xc0004c43e0, 0x2, 0x2)
/Users/rhari/go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:70 +0xdd fp=0xc0006e9c10 sp=0xc0006e9ba8 pc=0x1f776fd
github.com/keikoproj/active-monitor/controllers.(*HealthCheckReconciler).parseWorkflowFromHealthcheck(0xc00073c780, 0x23f72a0, 0xc0004781c0, 0xc000154a00, 0xc0005262a0, 0x0, 0x0)
/Users/rhari/go/src/github.com/keikoproj/active-monitor/controllers/healthcheck_controller.go:851 +0x5d0 fp=0xc0006e9de0 sp=0xc0006e9c10 pc=0x1f36010
github.com/keikoproj/active-monitor/controllers.(*HealthCheckReconciler).createSubmitWorkflow(0xc00073c780, 0x23ef140, 0xc0001341f8, 0x23f72a0, 0xc0004781c0, 0xc000154a00, 0x0, 0xc000b00000, 0xc000001e00, 0x0)
/Users/rhari/go/src/github.com/keikoproj/active-monitor/controllers/healthcheck_controller.go:471 +0x8c fp=0xc0006e9f00 sp=0xc0006e9de0 pc=0x1f2f30c
github.com/keikoproj/active-monitor/controllers.(*HealthCheckReconciler).createSubmitWorkflowHelper.func1()
/Users/rhari/go/src/github.com/keikoproj/active-monitor/controllers/healthcheck_controller.go:456 +0x115 fp=0xc0006e9fe0 sp=0xc0006e9f00 pc=0x1f3b535
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc0006e9fe8 sp=0xc0006e9fe0 pc=0x106d0a1
created by time.goFunc
/usr/local/go/src/time/sleep.go:167 +0x45
To Reproduce
Run Multiple workflows in parallel.
Expected behavior
Active-Monitor continue to work without issues.
Version
latest changes.