Details
-
Type:
Bug
-
Status: Resolved
-
Priority:
Major
-
Resolution: Fixed
-
Affects Version/s: 0.3
-
Fix Version/s: None
-
Component/s: REPL
-
Labels:None
Description
livy-server failed session because of some RPC issues in build 936.1. log
16/11/30 07:08:55.676 RSCClient-4-3: Received result for 31dfa59d-975a-49c1-890d-5b30288fad06 16/11/30 07:08:55.677 RSCClient-4-3: InteractiveSession 3 session state change from starting to running 16/11/30 07:08:58.807 RSCClient-4-3: [ClientProtocol] Failed to find handler for msg 'com.cloudera.livy.rsc.rpc.Rpc$MessageHeader'. 16/11/30 07:08:58.813 RSCClient-4-3: [ClientProtocol] Closing channel due to exception in pipeline (null). 16/11/30 07:08:58.813 RSCClient-4-3: Client RPC channel closed unexpectedly. 16/11/30 07:08:58.813 RSCClient-4-3: Error stopping RPC. io.netty.util.concurrent.BlockingOperationException: DefaultChannelPromise@47fae43(uncancellable) at io.netty.util.concurrent.DefaultPromise.checkDeadLock(DefaultPromise.java:390) at io.netty.channel.DefaultChannelPromise.checkDeadLock(DefaultChannelPromise.java:157) at io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:251) at io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:129) at io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:28) at io.netty.util.concurrent.DefaultPromise.sync(DefaultPromise.java:218) at io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:117) at io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:28) at com.cloudera.livy.rsc.rpc.Rpc.close(Rpc.java:305) at com.cloudera.livy.rsc.RSCClient.stop(RSCClient.java:221) at com.cloudera.livy.rsc.RSCClient$2$1.onSuccess(RSCClient.java:118) at com.cloudera.livy.rsc.RSCClient$2$1.onSuccess(RSCClient.java:112) at com.cloudera.livy.rsc.Utils$2.operationComplete(Utils.java:108) at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:680) at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:567) at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:406) at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:956) at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:608) at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:586) at io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1107) at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:543) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:528) at io.netty.channel.ChannelDuplexHandler.close(ChannelDuplexHandler.java:73) at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:543) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:528) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:394) at com.cloudera.livy.rsc.rpc.RpcDispatcher.exceptionCaught(RpcDispatcher.java:182) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:246) at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:737) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:310) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:244) at io.netty.handler.codec.ByteToMessageCodec.channelRead(ByteToMessageCodec.java:103) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:846) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) at java.lang.Thread.run(Thread.java:745) 16/11/30 07:08:58.817 RSCClient-4-3: [ClientProtocol] Expected RPC header, got com.cloudera.livy.rsc.ReplJobResults instead. 16/11/30 07:08:58.817 RSCClient-4-3: [ClientProtocol] Closing channel due to exception in pipeline (null). 16/11/30 07:08:58.817 RSCClient-4-3: [ClientProtocol] Closing RPC channel with 1 outstanding RPCs. 16/11/30 07:08:58.821 qtp1554249927-29: internal error java.util.concurrent.ExecutionException: java.util.concurrent.CancellationException at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:37) at com.cloudera.livy.server.interactive.InteractiveSession.executeStatement(InteractiveSession.scala:463) at com.cloudera.livy.server.interactive.InteractiveSessionServlet$$anonfun$11$$anonfun$apply$6.apply(InteractiveSessionServlet.scala:124) at com.cloudera.livy.server.interactive.InteractiveSessionServlet$$anonfun$11$$anonfun$apply$6.apply(InteractiveSessionServlet.scala:123) at com.cloudera.livy.server.SessionServlet.doWithSession(SessionServlet.scala:194) at com.cloudera.livy.server.SessionServlet.withSession(SessionServlet.scala:187) at com.cloudera.livy.server.interactive.InteractiveSessionServlet$$anonfun$11.apply(InteractiveSessionServlet.scala:123) at com.cloudera.livy.server.interactive.InteractiveSessionServlet$$anonfun$11.apply(InteractiveSessionServlet.scala:122) at com.cloudera.livy.server.JsonServlet.com$cloudera$livy$server$JsonServlet$$doAction(JsonServlet.scala:114) at com.cloudera.livy.server.JsonServlet$$anonfun$jpost$1.apply(JsonServlet.scala:76) at org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$$liftAction(ScalatraBase.scala:270) at org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:265) at org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:265) at org.scalatra.ApiFormats$class.withRouteMultiParams(ApiFormats.scala:178) at com.cloudera.livy.server.JsonServlet.withRouteMultiParams(JsonServlet.scala:40) at org.scalatra.ScalatraBase$class.invoke(ScalatraBase.scala:264) at org.scalatra.ScalatraServlet.invoke(ScalatraServlet.scala:49) at org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:240) at org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:238) at scala.Option.flatMap(Option.scala:170) at org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:238) at org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:237) at scala.collection.immutable.Stream.flatMap(Stream.scala:446) at org.scalatra.ScalatraBase$class.runRoutes(ScalatraBase.scala:237) at org.scalatra.ScalatraServlet.runRoutes(ScalatraServlet.scala:49) at org.scalatra.ScalatraBase$class.runActions$1(ScalatraBase.scala:163) at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply$mcV$sp(ScalatraBase.scala:175) at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply(ScalatraBase.scala:175) at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply(ScalatraBase.scala:175) at org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$$cradleHalt(ScalatraBase.scala:193) at org.scalatra.ScalatraBase$class.executeRoutes(ScalatraBase.scala:175) at org.scalatra.ScalatraServlet.executeRoutes(ScalatraServlet.scala:49) at org.scalatra.ScalatraBase$$anonfun$handle$1.apply$mcV$sp(ScalatraBase.scala:113) at org.scalatra.ScalatraBase$$anonfun$handle$1.apply(ScalatraBase.scala:113) at org.scalatra.ScalatraBase$$anonfun$handle$1.apply(ScalatraBase.scala:113) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:57) at org.scalatra.DynamicScope$class.withResponse(DynamicScope.scala:80) at org.scalatra.ScalatraServlet.withResponse(ScalatraServlet.scala:49) at org.scalatra.DynamicScope$$anonfun$withRequestResponse$1.apply(DynamicScope.scala:60) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:57) at org.scalatra.DynamicScope$class.withRequest(DynamicScope.scala:71) at org.scalatra.ScalatraServlet.withRequest(ScalatraServlet.scala:49) at org.scalatra.DynamicScope$class.withRequestResponse(DynamicScope.scala:59) at org.scalatra.ScalatraServlet.withRequestResponse(ScalatraServlet.scala:49) at org.scalatra.ScalatraBase$class.handle(ScalatraBase.scala:111) at org.scalatra.ScalatraServlet.org$scalatra$servlet$ServletBase$$super$handle(ScalatraServlet.scala:49) at org.scalatra.servlet.ServletBase$class.handle(ServletBase.scala:43) at com.cloudera.livy.server.SessionServlet.org$scalatra$MethodOverride$$super$handle(SessionServlet.scala:40) at org.scalatra.MethodOverride$class.handle(MethodOverride.scala:28) at com.cloudera.livy.server.interactive.InteractiveSessionServlet.org$scalatra$servlet$FileUploadSupport$$super$handle(InteractiveSessionServlet.scala:42) at org.scalatra.servlet.FileUploadSupport$class.handle(FileUploadSupport.scala:93) at com.cloudera.livy.server.interactive.InteractiveSessionServlet.handle(InteractiveSessionServlet.scala:42) at org.scalatra.ScalatraServlet.service(ScalatraServlet.scala:54) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) at org.eclipse.jetty.server.Server.handle(Server.java:499) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) Caused by: java.util.concurrent.CancellationException 16/11/30 07:08:58.867 ForkJoinPool-1-worker-3: Stopping InteractiveSession 3... 16/11/30 07:08:58.868 ForkJoinPool-1-worker-3: InteractiveSession 3 session state change from running to shutting_down