sky-uk/kafka-message-scheduler

Investigate KMS failing at startup

manuelcueto opened this issue · 2 comments

manuelcueto commented

Kms dies at startup .

Investigate

manuelcueto commented

KMS 0.21.0 constantly restarting in both dev gcp and dcos with the following error after loading topics:

Might be worth adding a docker compose test to the pr build?

{
    "@timestamp": "2019-08-12T14:43:39.936Z",
    "@version": "1",
    "message": "Reader stream has died",
    "logger_name": "com.sky.kms.actors.SchedulingActor",
    "thread_name": "kafka-message-scheduler-akka.actor.default-dispatcher-31",
    "level": "ERROR",
    "level_value": 40000,
    "stack_trace": "java.util.concurrent.TimeoutException: No elements passed in the last 5 minutes.
        at akka.stream.impl.Timers$Idle$$anon$3.onTimer(Timers.scala:110)
        at akka.stream.stage.TimerGraphStageLogic.onInternalTimer(GraphStage.scala:1544)
        at akka.stream.stage.TimerGraphStageLogic.$anonfun$getTimerAsyncCallback$1(GraphStage.scala:1533)
        at akka.stream.stage.TimerGraphStageLogic.$anonfun$getTimerAsyncCallback$1$adapted(GraphStage.scala:1533)
        at akka.stream.impl.fusing.GraphInterpreter.runAsyncInput(GraphInterpreter.scala:452)
        at akka.stream.impl.fusing.GraphInterpreterShell$AsyncInput.execute(ActorGraphInterpreter.scala:481)
        at akka.stream.impl.fusing.GraphInterpreterShell.processEvent(ActorGraphInterpreter.scala:581)
        at akka.stream.impl.fusing.ActorGraphInterpreter.akka$stream$impl$fusing$ActorGraphInterpreter$$processEvent(ActorGraphInterpreter.scala:749)
        at akka.stream.impl.fusing.ActorGraphInterpreter$$anonfun$receive$1.applyOrElse(ActorGraphInterpreter.scala:764)
        at akka.actor.Actor.aroundReceive(Actor.scala:539)
        at akka.actor.Actor.aroundReceive$(Actor.scala:537)
        at akka.stream.impl.fusing.ActorGraphInterpreter.aroundReceive(ActorGraphInterpreter.scala:671)
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:612)
        at akka.actor.ActorCell.invoke_aroundBody0(ActorCell.scala:581)
        at akka.actor.ActorCell$AjcClosure1.run(ActorCell.scala:1)
        at org.aspectj.runtime.reflect.JoinPointImpl.proceed(JoinPointImpl.java:149)
        at akka.kamon.instrumentation.ActorMonitors$$anon$1.$anonfun$processMessage$1(ActorMonitor.scala:134)
        at kamon.Kamon$.withContext(Kamon.scala:120)
        at akka.kamon.instrumentation.ActorMonitors$$anon$1.processMessage(ActorMonitor.scala:134)
        at akka.kamon.instrumentation.ActorCellInstrumentation.aroundBehaviourInvoke(ActorInstrumentation.scala:45)
        at akka.actor.ActorCell.invoke(ActorCell.scala:574)
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
        at akka.dispatch.Mailbox.run(Mailbox.scala:229)
        at kamon.executors.Executors$InstrumentedExecutorService$$anon$7.run(Executors.scala:270)
        at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
        at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
        at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
        at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
        at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)",
    "sourceThread": "kafka-message-scheduler-akka.actor.default-dispatcher-25",
    "akkaSource": "akka://kafka-message-scheduler/user/scheduling-actor",
    "sourceActorSystem": "kafka-message-scheduler",
    "akkaTimestamp": "14:43:39.918UTC"
}