[Bug]: Occasional CosmosDB panic in handleBatchSizeChange
Closed this issue · 0 comments
arielshaqed commented
What happened?
What actually happened, including error codes if applies.
Esti failed on Azure ADLS Gen2 because lakeFS panicked.
Steps to Reproduce:
This is a flake. It still should not have happened. See this action run, search for "panic"
Expected behavior
lakeFS should not crash.
lakeFS version
latest HEAD
How lakeFS is installed
Esti
Affected clients
No response
Relevant log output
lakefs-1 | time="2024-11-12T16:50:53Z" level=debug msg="Filtering actions" func="pkg/actions.(*StoreService).Run" file="build/pkg/actions/service.go:285" host="lakefs:8000" method=POST operation_id=CreateBranch path=/api/v1/repositories/testhooksfail-create-branch/branches record="{580h94oqemhc738efde0 pre-create-branch testhooksfail-create-branch https://esti4hns.blob.core.windows.net/esti-system-testing/13232/25356/cspoer0qemhegouq9kl0/testhooksfail/create_branch feature-1 test_branch {0 0001-01-01 00:00:00 +0000 UTC [] map[] 0} 034a98a09a3f0eb980c435eea6563b57844b5b54f8cf59f163e8d2da13a42481 }" spec="{pre-create-branch test_branch}" user=esti
lakefs-1 | 2024/11/12 16:50:58 http: panic serving 172.18.0.3:58136: runtime error: invalid memory address or nil pointer dereference
lakefs-1 | goroutine 3393 [running]:
lakefs-1 | net/http.(*conn).serve.func1()
lakefs-1 | /usr/local/go/src/net/http/server.go:1903 +0xbe
lakefs-1 | panic({0x22c4ac0?, 0x72ac350?})
lakefs-1 | /usr/local/go/src/runtime/panic.go:770 +0x132
lakefs-1 | github.com/treeverse/lakefs/pkg/kv/cosmosdb.(*EntriesIterator).handleBatchSizeChange(...)
lakefs-1 | /build/pkg/kv/cosmosdb/store.go:461
lakefs-1 | github.com/treeverse/lakefs/pkg/kv/cosmosdb.(*EntriesIterator).Next(0xc0081c3500)
lakefs-1 | /build/pkg/kv/cosmosdb/store.go:424 +0xf4
lakefs-1 | github.com/treeverse/lakefs/pkg/kv.(*PartitionIterator).Next(0xc0002a90a0)
lakefs-1 | /build/pkg/kv/iterators.go:282 +0x171
lakefs-1 | github.com/treeverse/lakefs/pkg/graveler/staging.(*Iterator).Next(0xc008240780)
lakefs-1 | /build/pkg/graveler/staging/iterator.go:30 +0x37
lakefs-1 | github.com/treeverse/lakefs/pkg/graveler.(*CombinedIterator).advanceInnerIterators(0xc0082407b0)
lakefs-1 | /build/pkg/graveler/combined_iterator.go:51 +0x1dd
lakefs-1 | github.com/treeverse/lakefs/pkg/graveler.(*CombinedIterator).Next(0xc0082407b0)
lakefs-1 | /build/pkg/graveler/combined_iterator.go:87 +0x1c
lakefs-1 | github.com/treeverse/lakefs/pkg/graveler.(*FilterTombstoneIterator).Next(0xc007970fc0)
lakefs-1 | /build/pkg/graveler/combined_iterator.go:144 +0x2a
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*valueEntryIterator).Next(0xc0082407e0)
lakefs-1 | /build/pkg/catalog/value_entry_iterator.go:23 +0x32
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*prefixIterator).Next(0xc008240ba0)
lakefs-1 | /build/pkg/catalog/prefix_iterator.go:28 +0x2e
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*entryListingIterator).nextWithDelimiter(0xc007aeaac0)
lakefs-1 | /build/pkg/catalog/entry_listing_iterator.go:75 +0x5f
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*entryListingIterator).Next(0x6057bc0?)
lakefs-1 | /build/pkg/catalog/entry_listing_iterator.go:36 +0x13
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*Catalog).ListEntries(0xc00017e840, {0x6053130, 0xc0081d4540}, {0xc00778f880, 0x1b}, {0xc007106ff0, 0x9}, {0x270a4f0, 0x10}, {0x0, ...}, ...)
lakefs-1 | /build/pkg/catalog/catalog.go:1189 +0x570
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*ActionsSource).list(_, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, {0xc0079a5b80, ...}, ...})
lakefs-1 | /build/pkg/catalog/actions_source.go:59 +0xce
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*ActionsSource).List.func1()
lakefs-1 | /build/pkg/catalog/actions_source.go:43 +0x58
lakefs-1 | github.com/treeverse/lakefs/pkg/cache.(*GetSetCache).GetOrSet.func1()
lakefs-1 | /build/pkg/cache/cache.go:42 +0x13
lakefs-1 | github.com/treeverse/lakefs/pkg/cache.(*GetSetCache).GetOrSetWithExpiry.func1()
lakefs-1 | /build/pkg/cache/cache.go:53 +0x39
lakefs-1 | github.com/treeverse/lakefs/pkg/cache.(*ChanOnlyOne).Compute(0xc000130080, {0x219e520, 0xc007970dd0}, 0xc00766a150)
lakefs-1 | /build/pkg/cache/only_one.go:35 +0xdb
lakefs-1 | github.com/treeverse/lakefs/pkg/cache.(*GetSetCache).GetOrSetWithExpiry(0xc00088e040, {0x219e520, 0xc007970dd0}, 0xc00766a1a8)
lakefs-1 | /build/pkg/cache/cache.go:52 +0x93
lakefs-1 | github.com/treeverse/lakefs/pkg/cache.(*GetSetCache).GetOrSet(0xc009f32270?, {0x219e520?, 0xc007970dd0?}, 0x2?)
lakefs-1 | /build/pkg/cache/cache.go:45 +0x39
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*ActionsSource).List(_, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, {0xc0079a5b80, ...}, ...})
lakefs-1 | /build/pkg/catalog/actions_source.go:42 +0x1b1
lakefs-1 | github.com/treeverse/lakefs/pkg/actions.LoadActions({_, _}, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, ...})
lakefs-1 | /build/pkg/actions/action.go:183 +0x10e
lakefs-1 | github.com/treeverse/lakefs/pkg/actions.(*StoreService).loadMatchedActions(_, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, {0xc0079a5b80, ...}, ...}, ...)
lakefs-1 | /build/pkg/actions/service.go:309 +0xa5
lakefs-1 | github.com/treeverse/lakefs/pkg/actions.(*StoreService).Run(_, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, {0xc0079a5b80, ...}, ...})
lakefs-1 | /build/pkg/actions/service.go:286 +0x2d0
lakefs-1 | github.com/treeverse/lakefs/pkg/actions.(*StoreService).PreCreateBranchHook(_, {_, _}, {{0xc0084782e8, 0x14}, {0x270c836, 0x11}, {0xc00778f880, 0x1b}, {0xc0079a5b80, ...}, ...})
lakefs-1 | /build/pkg/actions/service.go:569 +0x5a
lakefs-1 | github.com/treeverse/lakefs/pkg/graveler.(*Graveler).CreateBranch(0xc000574090, {0x6053130, 0xc0081d4540}, 0xc0075d9bd8, {0xc007106fe0, 0xb}, {0xc007106ff0, 0x9}, {0xc00aeaee30, 0x1, ...})
lakefs-1 | /build/pkg/graveler/graveler.go:1259 +0x509
lakefs-1 | github.com/treeverse/lakefs/pkg/catalog.(*Catalog).CreateBranch(0xc00017e840, {0x6053130, 0xc0081d4540}, {0xc0079eea6a, 0x1b}, {0xc007106fe0, 0xb}, {0xc007106ff0, 0x9}, {0xc00aeaee30, ...})
lakefs-1 | /build/pkg/catalog/catalog.go:695 +0x3cf
lakefs-1 | github.com/treeverse/lakefs/pkg/api.(*Controller).CreateBranch(0xc00716ed20, {0x604d388, 0xc007d805a0}, 0xc007baf680, {0x0, {0xc007106fe0, 0xb}, {0xc007106ff0, 0x9}}, {0xc0079eea6a, ...})
lakefs-1 | /build/pkg/api/controller.go:2613 +0x25f
lakefs-1 | github.com/treeverse/lakefs/pkg/api/apigen.(*ServerInterfaceWrapper).CreateBranch.func1({0x604d388?, 0xc007d805a0?}, 0x219e520?)
lakefs-1 | /build/pkg/api/apigen/lakefs.gen.go:26460 +0x6a
lakefs-1 | github.com/treeverse/lakefs/pkg/api/apigen.(*ServerInterfaceWrapper).CreateBranch(0xc0075baba0, {0x604d388, 0xc007d805a0}, 0xc007baf440)
lakefs-1 | /build/pkg/api/apigen/lakefs.gen.go:26467 +0x56f
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0xc0075ac770?, {0x604d388?, 0xc007d805a0?}, 0xc0081d43c0?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/treeverse/lakefs/pkg/api.MetricsMiddleware.func1.1({0x7f48cf10b450, 0xc0081d43c0}, 0xc007baf440)
lakefs-1 | /build/pkg/api/metrics_middleware.go:26 +0xe7
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0x72b6690?, {0x7f48cf10b450?, 0xc0081d43c0?}, 0x120?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x604d448?, 0xc00748df60?}, 0xc007baf440)
lakefs-1 | /go/pkg/mod/github.com/prometheus/client_golang@v1.17.0/prometheus/promhttp/instrument_server.go:147 +0xc3
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0x6053130?, {0x604d448?, 0xc00748df60?}, 0x6028280?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/treeverse/lakefs/pkg/api.AuthMiddleware.func1.1({0x604d448, 0xc00748df60}, 0xc007baed80)
lakefs-1 | /build/pkg/api/auth_middleware.go:111 +0x4a2
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0xc008195e60?, {0x604d448?, 0xc00748df60?}, 0xc009da18c0?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/treeverse/lakefs/pkg/httputil.LoggingMiddleware.DefaultLoggingMiddleware.func2.1({0x604d268, 0xc007bb62a0}, 0xc007baea20)
lakefs-1 | /build/pkg/httputil/logging.go:98 +0x472
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0xc007baea20?, {0x604d268?, 0xc007bb62a0?}, 0xc007523eb0?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/treeverse/lakefs/pkg/api.OapiRequestValidatorWithOptions.func1.1({0x604d268, 0xc007bb62a0}, 0xc007bae6c0)
lakefs-1 | /build/pkg/api/serve.go:130 +0x2d4
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0xc0075e8320?, {0x604d268?, 0xc007bb62a0?}, 0xc0070523a0?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0x22a33e0?, {0x604d268?, 0xc007bb62a0?}, 0xc0079eea55?)
lakefs-1 | /go/pkg/mod/github.com/go-chi/chi/v5@v5.0.10/chain.go:31 +0x26
lakefs-1 | github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0008689c0, {0x604d268, 0xc007bb62a0}, 0xc007bae6c0)
lakefs-1 | /go/pkg/mod/github.com/go-chi/chi/v5@v5.0.10/mux.go:444 +0x2b4
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0x6053168?, {0x604d268?, 0xc007bb62a0?}, 0x72ad3e0?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0008689c0, {0x604d268, 0xc007bb62a0}, 0xc007bae5a0)
lakefs-1 | /go/pkg/mod/github.com/go-chi/chi/v5@v5.0.10/mux.go:90 +0x2ee
lakefs-1 | github.com/treeverse/lakefs/cmd/lakefs/cmd.init.func9.3({0x604d268, 0xc007bb62a0}, 0xc007bae5a0)
lakefs-1 | /build/cmd/lakefs/cmd/run.go:372 +0xfb
lakefs-1 | net/http.HandlerFunc.ServeHTTP(0x4146a5?, {0x604d268?, 0xc007bb62a0?}, 0xc007bb6201?)
lakefs-1 | /usr/local/go/src/net/http/server.go:2171 +0x29
lakefs-1 | net/http.serverHandler.ServeHTTP({0x603d658?}, {0x604d268?, 0xc007bb62a0?}, 0x6?)
lakefs-1 | /usr/local/go/src/net/http/server.go:3142 +0x8e
lakefs-1 | net/http.(*conn).serve(0xc00a6b9c20, {0x6053130, 0xc000b6c030})
lakefs-1 | /usr/local/go/src/net/http/server.go:2044 +0x5e8
lakefs-1 | created by net/http.(*Server).Serve in goroutine 103
lakefs-1 | /usr/local/go/src/net/http/server.go:3290 +0x4b4
esti-1 | hooks_failure_test.go:107:
esti-1 | Error Trace: /lakefs/esti/hooks_failure_test.go:107
esti-1 | /lakefs/esti/hooks_failure_test.go:58
esti-1 | Error: Received unexpected error:
esti-1 | Post "http://lakefs:8000/api/v1/repositories/testhooksfail-create-branch/branches": EOF
esti-1 | Test: TestHooksFail/create_branch
esti-1 | --- FAIL: TestHooksFail (6.98s)
esti-1 | --- PASS: TestHooksFail/commit (1.33s)
esti-1 | --- FAIL: TestHooksFail/create_branch (5.65s)Contact details
ariels