Pod stuck in crashloopbackoff
jdziedzic opened this issue · 0 comments
We upgraded to 1.3.4 and are now seeing the pod constantly crash-looping. This is the last error in the log before the pod gets killed.
2022-02-15T14:48:51.250Z DPANIC controllers.secret_to_keystore_contoller odd number of arguments passed as key-value pairs for logging {"ignored key": "keystore"}
github.com/go-logr/zapr.handleFields
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100
github.com/go-logr/zapr.(*zapLogger).Error
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(*SecretToKeyStoreReconciler).Reconcile
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.2
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155
k8s.io/apimachinery/pkg/util/wait.BackoffUntil
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156
k8s.io/apimachinery/pkg/util/wait.JitterUntil
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185
k8s.io/apimachinery/pkg/util/wait.UntilWithContext
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99
E0215 14:48:51.250706 1 runtime.go:76] Observed a panic: odd number of arguments passed as key-value pairs for logging
goroutine 704 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x14374a0, 0xc0046d6320})
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:74 +0x85
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc02eb69420})
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:48 +0x75
panic({0x14374a0, 0xc0046d6320})
/opt/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1038 +0x215
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc02e6de160, {0xc051274500, 0x1, 0x1})
/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/entry.go:230 +0x486
go.uber.org/zap.(*Logger).DPanic(0x162b539, {0x1679925, 0x14374a0}, {0xc051274500, 0x1, 0x1})
/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/logger.go:215 +0x59
github.com/go-logr/zapr.handleFields(0xc0003a1e60, {0xc0046d62e0, 0x1, 0xe}, {0xc051274480, 0x1, 0x40e9f4})
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100 +0x535
github.com/go-logr/zapr.(*zapLogger).Error(0xc000532230, {0x180cdc0, 0xc0046d62d0}, {0x162e99c, 0xc01bcf4960}, {0xc0046d62e0, 0x1, 0x1})
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133 +0x1eb
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary({0xc054d1f400, 0x2077, 0x2077}, {0xc0522c2000, 0x1abd, 0x2cda}, {0xc0313d7b00, 0x9, 0x20}, {0x18536c0, ...})
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170 +0x2ff
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(*SecretToKeyStoreReconciler).Reconcile(0xc0003a1ec0, {0x1837d38, 0xc01bcf44e0}, {{{0xc01c052d25, 0xb}, {0xc01c077830, 0x22}}})
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127 +0x4cd
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc0006e6640, {0x1837c90, 0xc00086b980}, {0x14d6ac0, 0xc02eb69420})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x303
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc0006e6640, {0x1837c90, 0xc00086b980})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.2({0x1837c90, 0xc00086b980})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x46
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x25
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7efeb817dfd8)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0, {0x180eb80, 0xc02ff8f020}, 0x1, 0xc00091c060)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1, 0x3b9aca00, 0x0, 0x0, 0x1f4)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x1837c90, 0xc00086b980}, 0xc02ff8d1e0, 0xc0003ff560, 0xc00097f580, 0x90)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x99
k8s.io/apimachinery/pkg/util/wait.UntilWithContext({0x1837c90, 0xc00086b980}, 0xc02b771b80, 0xc0008b07a0)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x2b
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x356
panic: odd number of arguments passed as key-value pairs for logging [recovered]
panic: odd number of arguments passed as key-value pairs for logging
goroutine 704 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc02eb69420})
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:55 +0xd8
panic({0x14374a0, 0xc0046d6320})
/opt/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1038 +0x215
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc02e6de160, {0xc051274500, 0x1, 0x1})
/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/entry.go:230 +0x486
go.uber.org/zap.(*Logger).DPanic(0x162b539, {0x1679925, 0x14374a0}, {0xc051274500, 0x1, 0x1})
/home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/logger.go:215 +0x59
github.com/go-logr/zapr.handleFields(0xc0003a1e60, {0xc0046d62e0, 0x1, 0xe}, {0xc051274480, 0x1, 0x40e9f4})
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100 +0x535
github.com/go-logr/zapr.(*zapLogger).Error(0xc000532230, {0x180cdc0, 0xc0046d62d0}, {0x162e99c, 0xc01bcf4960}, {0xc0046d62e0, 0x1, 0x1})
/home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133 +0x1eb
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary({0xc054d1f400, 0x2077, 0x2077}, {0xc0522c2000, 0x1abd, 0x2cda}, {0xc0313d7b00, 0x9, 0x20}, {0x18536c0, ...})
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170 +0x2ff
github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(*SecretToKeyStoreReconciler).Reconcile(0xc0003a1ec0, {0x1837d38, 0xc01bcf44e0}, {{{0xc01c052d25, 0xb}, {0xc01c077830, 0x22}}})
/home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127 +0x4cd
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc0006e6640, {0x1837c90, 0xc00086b980}, {0x14d6ac0, 0xc02eb69420})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x303
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc0006e6640, {0x1837c90, 0xc00086b980})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.2({0x1837c90, 0xc00086b980})
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x46
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x25
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7efeb817dfd8)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0, {0x180eb80, 0xc02ff8f020}, 0x1, 0xc00091c060)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1, 0x3b9aca00, 0x0, 0x0, 0x1f4)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x1837c90, 0xc00086b980}, 0xc02ff8d1e0, 0xc0003ff560, 0xc00097f580, 0x90)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x99
k8s.io/apimachinery/pkg/util/wait.UntilWithContext({0x1837c90, 0xc00086b980}, 0xc02b771b80, 0xc0008b07a0)
/home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x2b
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1
/home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x356