Core dump when silo rolling update in K8s.
vrecluse opened this issue · 5 comments
vrecluse commented
Silo runs in k8s pod with UseConsoleLifetime() and UseKubernetesHosting().
When the silo pod rolling updates, core dump generated.
The stack:
Child SP IP Call Site
00007FE0FA2E4BA8 00007fe3b68ec747 [HelperMethodFrame_2OBJ: 00007fe0fa2e4ba8] System.Environment.FailFast(System.String, System.Exception)
00007FE0FA2E4CE0 00007FE341A5F90D Orleans.Runtime.FatalErrorHandler.OnFatalException(System.Object, System.String, System.Exception) [/_/src/Orleans.Runtime/Core/FatalErrorHandler.cs @ 46]
00007FE0FA2E4DC0 00007FE341A5F4F0 Orleans.Runtime.MembershipService.MembershipTableManager.KillMyselfLocally(System.String) [/_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs @ 618]
00007FE0FA2E4E10 00007FE3419E55EF Orleans.Runtime.MembershipService.MembershipTableManager+<CleanupMyTableEntries>d__52.MoveNext() [/_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs @ 550]
00007FE0FA2E4EF0 00007FE3419E51D7 System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[[Orleans.Runtime.MembershipService.MembershipTableManager+<CleanupMyTableEntries>d__52, Orleans.Runtime]](<CleanupMyTableEntries>d__52 ByRef) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncMethodBuilderCore.cs @ 38]
00007FE0FA2E4F30 00007FE3419E5125 Orleans.Runtime.MembershipService.MembershipTableManager.CleanupMyTableEntries(Orleans.MembershipTableData)
00007FE0FA2E4FB0 00007FE34138DB1C Orleans.Runtime.MembershipService.MembershipTableManager+<RefreshInternal>d__39.MoveNext() [/_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs @ 144]
00007FE0FA2E5020 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5070 00007FE3419E38FE System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.Boolean, System.Private.CoreLib],[Orleans.Runtime.MembershipService.MembershipTableManager+<RefreshInternal>d__39, Orleans.Runtime]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E50C0 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E5110 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5190 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E51C0 00007FE34138D7B9 Orleans.Providers.MongoDB.Membership.MongoMembershipTable+<DoAndLog>d__15`1[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E5290 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E52E0 00007FE3419E369A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[Orleans.Providers.MongoDB.Membership.MongoMembershipTable+<DoAndLog>d__15`1[[System.__Canon, System.Private.CoreLib]], Orleans.Providers.MongoDB]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E5330 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E5380 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5400 00007FE34138D2C6 Orleans.Providers.MongoDB.Membership.Store.Single.SingleMembershipCollection+<ReadAll>d__7.MoveNext()
00007FE0FA2E54B0 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5500 00007FE3419E3136 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[Orleans.Providers.MongoDB.Membership.Store.Single.SingleMembershipCollection+<ReadAll>d__7, Orleans.Providers.MongoDB]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E5550 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E55A0 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5620 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E5650 00007FE33F7CFE2C MongoDB.Driver.IAsyncCursorSourceExtensions+<FirstOrDefaultAsync>d__5`1[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E56E0 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5730 00007FE3411EAA1A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.IAsyncCursorSourceExtensions+<FirstOrDefaultAsync>d__5`1[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E5780 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E57D0 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5850 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E5880 00007FE33F795A70 MongoDB.Driver.MongoCollectionImpl`1+<UsingImplicitSessionAsync>d__107`1[[System.__Canon, System.Private.CoreLib],[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E5910 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5960 00007FE33F7CF92A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.MongoCollectionImpl`1+<UsingImplicitSessionAsync>d__107`1[[System.__Canon, System.Private.CoreLib],[System.__Canon, System.Private.CoreLib]], MongoDB.Driver]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E59B0 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E5A00 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5A80 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E5AB0 00007FE33F7CF6E3 MongoDB.Driver.MongoCollectionImpl`1+<ExecuteReadOperationAsync>d__99`1[[System.__Canon, System.Private.CoreLib],[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E5B20 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5B70 00007FE3411E939A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.MongoCollectionImpl`1+<ExecuteReadOperationAsync>d__99`1[[System.__Canon, System.Private.CoreLib],[System.__Canon, System.Private.CoreLib]], MongoDB.Driver]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E5BC0 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E5C10 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5C90 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E5CC0 00007FE33F7CF3E8 MongoDB.Driver.OperationExecutor+<ExecuteReadOperationAsync>d__3`1[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E5D30 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5D80 00007FE3411E90DA System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.OperationExecutor+<ExecuteReadOperationAsync>d__3`1[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E5DD0 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E5E20 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E5EA0 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E5ED0 00007FE33F7CF11F MongoDB.Driver.Core.Operations.FindOperation`1+<ExecuteAsync>d__128[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E5F60 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E5FB0 00007FE3411E8E0A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Operations.FindOperation`1+<ExecuteAsync>d__128[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6000 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6050 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E60D0 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E6100 00007FE33F7CEBBD MongoDB.Driver.Core.Operations.FindOperation`1+<ExecuteAsync>d__129[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E6170 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E61C0 00007FE3411E7DBA System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Operations.FindOperation`1+<ExecuteAsync>d__129[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6210 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6260 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E62E0 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E6310 00007FE33F7CE829 MongoDB.Driver.Core.Operations.ReadCommandOperation`1+<ExecuteAsync>d__8[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E6390 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E63E0 00007FE3411E7AFA System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Operations.ReadCommandOperation`1+<ExecuteAsync>d__8[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6430 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6480 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6500 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E6530 00007FE33F7CE344 MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor+<ExecuteAsync>d__3`1[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E65C0 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E6610 00007FE3411E783A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor+<ExecuteAsync>d__3`1[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6660 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E66B0 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6730 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E6760 00007FE33F795610 MongoDB.Driver.Core.Servers.Server+ServerChannel+<ExecuteProtocolAsync>d__20`1[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E67E0 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E6830 00007FE33F7CDC9A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Servers.Server+ServerChannel+<ExecuteProtocolAsync>d__20`1[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6880 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E68D0 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6950 00007FE33E768D62 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.__Canon, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<System.__Canon>, System.__Canon) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E6980 00007FE33F795222 MongoDB.Driver.Core.WireProtocol.CommandUsingCommandMessageWireProtocol`1+<ExecuteAsync>d__20[[System.__Canon, System.Private.CoreLib]].MoveNext()
00007FE0FA2E6A90 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E6AE0 00007FE33F7CD27A System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.WireProtocol.CommandUsingCommandMessageWireProtocol`1+<ExecuteAsync>d__20[[System.__Canon, System.Private.CoreLib]], MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6B30 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6B80 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6C00 00007FE33F794A1C MongoDB.Driver.Core.ConnectionPools.ExclusiveConnectionPool+PooledConnection+<ReceiveMessageAsync>d__26.MoveNext()
00007FE0FA2E6C70 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E6CC0 00007FE33F7CCFF6 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.ConnectionPools.ExclusiveConnectionPool+PooledConnection+<ReceiveMessageAsync>d__26, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6D10 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6D60 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6DE0 00007FE33F794654 MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveMessageAsync>d__48.MoveNext()
00007FE0FA2E6E60 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E6EB0 00007FE33F7CCD86 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveMessageAsync>d__48, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E6F00 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E6F50 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E6FD0 00007FE33F7940D4 MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveBufferAsync>d__46.MoveNext()
00007FE0FA2E7070 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E70C0 00007FE33F7CCB16 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveBufferAsync>d__46, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E7110 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E7160 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E71E0 00007FE33F792F7E MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveBufferAsync>d__45.MoveNext()
00007FE0FA2E72F0 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E7340 00007FE33F7CC1F6 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.__Canon, System.Private.CoreLib],[MongoDB.Driver.Core.Connections.BinaryConnection+<ReceiveBufferAsync>d__45, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E7390 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E73E0 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E7460 00007FE33F7CC8E0 MongoDB.Driver.Core.Misc.StreamExtensionMethods+<ReadBytesAsync>d__5.MoveNext()
00007FE0FA2E7540 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E7590 00007FE3419A370E System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.Threading.Tasks.VoidTaskResult, System.Private.CoreLib],[MongoDB.Driver.Core.Misc.StreamExtensionMethods+<ReadBytesAsync>d__5, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E75E0 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E7630 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E76B0 00007FE33F78596B System.Threading.Tasks.Task`1[[System.Int32, System.Private.CoreLib]].TrySetResult(Int32) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs @ 400]
00007FE0FA2E76E0 00007FE33F790D16 System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1[[System.Int32, System.Private.CoreLib]].SetExistingTaskResult(System.Threading.Tasks.Task`1<Int32>, Int32) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 446]
00007FE0FA2E7710 00007FE33F78C208 MongoDB.Driver.Core.Misc.StreamExtensionMethods+<ReadAsync>d__1.MoveNext()
00007FE0FA2E7780 00007FE33E74AC29 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ExecutionContext.cs @ 183]
00007FE0FA2E77D0 00007FE33F7CBD7E System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1+AsyncStateMachineBox`1[[System.Int32, System.Private.CoreLib],[MongoDB.Driver.Core.Misc.StreamExtensionMethods+<ReadAsync>d__1, MongoDB.Driver.Core]].MoveNext(System.Threading.Thread) [/_/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncTaskMethodBuilderT.cs @ 324]
00007FE0FA2E7820 00007FE33E757A8B System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Runtime.CompilerServices.IAsyncStateMachineBox, Boolean) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TaskContinuation.cs @ 795]
00007FE0FA2E7870 00007FE33E73A813 System.Threading.Tasks.Task.RunContinuations(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @ 3410]
00007FE0FA2E78F0 00007FE33F78596B System.Threading.Tasks.Task`1[[System.Int32, System.Private.CoreLib]].TrySetResult(Int32) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs @ 400]
00007FE0FA2E7920 00007FE33F7CBB43 System.Threading.Tasks.ValueTask`1+ValueTaskSourceAsTask+<>c[[System.Int32, System.Private.CoreLib]].<.cctor>b__4_0(System.Object) [/_/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/ValueTask.cs @ 649]
00007FE0FA2E7960 00007FE33F78A749 System.Net.Sockets.Socket+AwaitableSocketAsyncEventArgs.InvokeContinuation(System.Action`1<System.Object>, System.Object, Boolean, Boolean) [/_/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs @ 1321]
00007FE0FA2E79A0 00007FE33F78A5D5 System.Net.Sockets.Socket+AwaitableSocketAsyncEventArgs.OnCompleted(System.Net.Sockets.SocketAsyncEventArgs) [/_/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs @ 989]
00007FE0FA2E79F0 00007FE33F66F6C6 System.Net.Sockets.SocketAsyncEngine.System.Threading.IThreadPoolWorkItem.Execute()
00007FE0FA2E7A70 00007FE33E739EDC System.Threading.ThreadPoolWorkQueue.Dispatch() [/_/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs @ 919]
00007FE0FA2E7AE0 00007FE3413AFDAD System.Threading.PortableThreadPool+WorkerThread.WorkerThreadStart() [/_/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs @ 77]
00007FE0FA2E7CF0 00007fe3b64fde27 [DebuggerU2MCatchHandlerFrame: 00007fe0fa2e7cf0]
vrecluse commented
The related log enties of old silo and new silo:
Old silo received the SIGTERM signal:
[15:03:26 INF] Application is shutting down... <s:Microsoft.Hosting.Lifetime>
New silo starts, and marks old silo as dead:
[15:03:32 INF] Starting Orleans Silo. <s:Orleans.Hosting.SiloHostedService>
[15:03:33 WRN] Silo S172.160.126.177:11111:44953007/silo-server-5967d5944f-zzmtf/Active does not correspond to any known pod. Marking it as dead. <s:Orleans.Hosting.Kubernetes.KubernetesClusterAgent>
After a while, old silo cleanup is going to finish, but finds itself dead, then causes fatal error:
15:04:12 WRN] I should be Dead according to membership table (in CleanupTableEntries): entry = [SiloAddress=S172.160.126.177:11111:44953007 SiloName=silo-server-5967d5944f-zzmtf Status=Dead HostName=silo-server-5967d5944f-zzmtf ProxyPort=30000 RoleName=Panda.Silo.Server UpdateZone=0 FaultZone=0 StartTime=2023-06-05 06:56:47.375 GMT IAmAliveTime=2023-06-05 07:01:48.617 GMT Suspecters=[S172.160.108.170:11111:44953412] SuspectTimes=[2023-06-05 07:03:33.266 GMT]]. <s:Orleans.Runtime.MembershipService.MembershipTableManager>
[15:04:12 ERR] I have been told I am dead, so this silo will stop! Reason: I should be Dead according to membership table (in CleanupTableEntries): entry = [SiloAddress=S172.160.126.177:11111:44953007 SiloName=silo-server-5967d5944f-zzmtf Status=Dead HostName=silo-server-5967d5944f-zzmtf ProxyPort=30000 RoleName=Panda.Silo.Server UpdateZone=0 FaultZone=0 StartTime=2023-06-05 06:56:47.375 GMT IAmAliveTime=2023-06-05 07:01:48.617 GMT Suspecters=[S172.160.108.170:11111:44953412] SuspectTimes=[2023-06-05 07:03:33.266 GMT]]. <s:Orleans.Runtime.MembershipService.MembershipTableManager>
[15:04:12 ERR] Fatal error from Orleans.Runtime.MembershipService.MembershipTableManager. Context: I have been told I am dead, so this silo will stop! Reason: I should be Dead according to membership table (in CleanupTableEntries): entry = [SiloAddress=S172.160.126.177:11111:44953007 SiloName=silo-server-5967d5944f-zzmtf Status=Dead HostName=silo-server-5967d5944f-zzmtf ProxyPort=30000 RoleName=Panda.Silo.Server UpdateZone=0 FaultZone=0 StartTime=2023-06-05 06:56:47.375 GMT IAmAliveTime=2023-06-05 07:01:48.617 GMT Suspecters=[S172.160.108.170:11111:44953412] SuspectTimes=[2023-06-05 07:03:33.266 GMT]]. <s:Orleans.Runtime.FatalErrorHandler>
FATAL EXCEPTION from Orleans.Runtime.MembershipService.MembershipTableManager. Context: I have been told I am dead, so this silo will stop! Reason: I should be Dead according to membership table (in CleanupTableEntries): entry = [SiloAddress=S172.160.126.177:11111:44953007 SiloName=silo-server-5967d5944f-zzmtf Status=Dead HostName=silo-server-5967d5944f-zzmtf ProxyPort=30000 RoleName=Panda.Silo.Server UpdateZone=0 FaultZone=0 StartTime=2023-06-05 06:56:47.375 GMT IAmAliveTime=2023-06-05 07:01:48.617 GMT Suspecters=[S172.160.108.170:11111:44953412] SuspectTimes=[2023-06-05 07:03:33.266 GMT]].. Exception: null.\nCurrent stack: at System.Environment.get_StackTrace()
at Orleans.Runtime.FatalErrorHandler.OnFatalException(Object sender, String context, Exception exception) in /_/src/Orleans.Runtime/Core/FatalErrorHandler.cs:line 37
at Orleans.Runtime.MembershipService.MembershipTableManager.KillMyselfLocally(String reason) in /_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs:line 618
at Orleans.Runtime.MembershipService.MembershipTableManager.CleanupMyTableEntries(MembershipTableData table) in /_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs:line 550
at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine)
at Orleans.Runtime.MembershipService.MembershipTableManager.CleanupMyTableEntries(MembershipTableData table)
at Orleans.Runtime.MembershipService.MembershipTableManager.RefreshInternal(Boolean requireCleanup) in /_/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs:line 144
at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.AsyncStateMachineBox`1.ExecutionContextCallback(Object s)
at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)
at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.AsyncStateMachineBox`1.MoveNext(Thread threadPoolThread)
at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.AsyncStateMachineBox`1.MoveNext()
at System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(IAsyncStateMachineBox box, Boolean allowInlining)
at System.Threading.Tasks.Task.RunContinuations(Object continuationObject)
at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.SetExistingTaskResult(Task`1 task, TResult result)
at Orleans.Providers.MongoDB.Membership.MongoMembershipTable.DoAndLog[T](String actionName, Func`1 action)
at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.AsyncStateMachineBox`1.ExecutionContextCallback(Object s)
at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state)
...
vrecluse commented
The pod strategy is RollingUpdate.
ReubenBond commented
Good find. The new pod is killing the old pod before it the old pod gets a chance to shutdown. We should fix this. So, between when the silo starts shutting down until the time it detects that it has been marked as dead, 46 seconds elapse. I wonder what kind of leeway pods should have to terminate.
EDIT: which version of Orleans are you using?
vrecluse commented
I'm using Orleans 7.1.2 .
Tried to implement my CustomFatalErrorHandler like this:
public class CustomFatalErrorHandler : IFatalErrorHandler
{
private readonly ClusterMembershipOptions _clusterMembershipOptions;
private readonly ILogger<CustomFatalErrorHandler> _log;
private bool _isShuttingDown;
public CustomFatalErrorHandler(
ILogger<CustomFatalErrorHandler> log,
IOptions<ClusterMembershipOptions> clusterMembershipOptions, IHostApplicationLifetime applicationLifetime)
{
_log = log;
_clusterMembershipOptions = clusterMembershipOptions.Value;
_isShuttingDown = false;
applicationLifetime.ApplicationStopping.Register(() => _isShuttingDown = true);
}
public bool IsUnexpected(Exception exception)
{
return !(exception is ThreadAbortException);
}
public void OnFatalException(object sender, string context, Exception exception)
{
string msg = null;
if (!_isShuttingDown)
{
_log.LogError(
(int)ErrorCode.Logger_ProcessCrashing,
exception,
"Fatal error from {Sender}. Context: {Context}",
sender,
context);
Console.Error.WriteLine(msg);
msg = @$"FATAL EXCEPTION from {sender?.ToString() ?? "null"}. Context: {context ?? "null"
}. Exception: {(exception != null ? LogFormatter.PrintException(exception) : "null")}.\nCurrent stack: {Environment.StackTrace}";
}
else
{
_log.LogWarning(
exception,
"Exception when shutting down, from {Sender}. Context: {Context}",
sender,
context);
}
// Allow some time for loggers to flush.
Thread.Sleep(2000);
if (Debugger.IsAttached) Debugger.Break();
if (!_isShuttingDown)
Environment.FailFast(msg, exception);
}
}It doesn't core dump anymore, but the application does not terminate nomally, until terminationGracePeriodSeconds expired.