Pyspark session creation fails with NullPointerException

Logs:

21/11/18 14:35:18 INFO InteractiveSession$: Creating Interactive session 1962: [owner: null, request: [kind: pyspark, proxyUser: None, driverMemory: 4000M, executorCores: 2, heartbeatTimeoutInSecond: 0]]
21/11/18 14:35:18 ERROR SessionServlet$: internal error
java.lang.NullPointerException
    at org.apache.livy.rsc.ContextLauncher.<init>(ContextLauncher.java:89)
    at org.apache.livy.rsc.ContextLauncher.create(ContextLauncher.java:76)
    at org.apache.livy.rsc.RSCClientFactory.createClient(RSCClientFactory.java:72)
    at org.apache.livy.LivyClientBuilder.build(LivyClientBuilder.java:142)
    at org.apache.livy.server.interactive.InteractiveSession$$anonfun$1.apply(InteractiveSession.scala:111)
    at org.apache.livy.server.interactive.InteractiveSession$$anonfun$1.apply(InteractiveSession.scala:81)
    at scala.Option.orElse(Option.scala:289)
    at org.apache.livy.server.interactive.InteractiveSession$.create(InteractiveSession.scala:81)
    at org.apache.livy.server.interactive.InteractiveSessionServlet.createSession(InteractiveSessionServlet.scala:55)
    at org.apache.livy.server.interactive.InteractiveSessionServlet.createSession(InteractiveSessionServlet.scala:40)
    at org.apache.livy.server.SessionServlet$$anonfun$16.apply(SessionServlet.scala:134)
    at org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$$liftAction(ScalatraBase.scala:280)
    at org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:274)
    at org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:274)
    at org.scalatra.ApiFormats$class.withRouteMultiParams(ApiFormats.scala:183)
    at org.apache.livy.server.JsonServlet.withRouteMultiParams(JsonServlet.scala:39)
    at org.scalatra.ScalatraBase$class.invoke(ScalatraBase.scala:273)
    at org.apache.livy.server.JsonServlet.invoke(JsonServlet.scala:39)
    at org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:248)
    at org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:246)
    at scala.Option.flatMap(Option.scala:171)
    at org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:246)
    at org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:245)
    at scala.collection.immutable.Stream.flatMap(Stream.scala:493)
    at org.scalatra.ScalatraBase$class.runRoutes(ScalatraBase.scala:245)
    at org.apache.livy.server.JsonServlet.runRoutes(JsonServlet.scala:39)
    at org.scalatra.ScalatraBase$class.runActions$1(ScalatraBase.scala:169)
    at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply$mcV$sp(ScalatraBase.scala:181)
    at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply(ScalatraBase.scala:181)
    at org.scalatra.ScalatraBase$$anonfun$executeRoutes$1.apply(ScalatraBase.scala:181)
    at org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$$cradleHalt(ScalatraBase.scala:199)
    at org.scalatra.ScalatraBase$class.executeRoutes(ScalatraBase.scala:181)
    at org.apache.livy.server.JsonServlet.executeRoutes(JsonServlet.scala:39)
    at org.scalatra.ScalatraBase$$anonfun$handle$1.apply$mcV$sp(ScalatraBase.scala:123)
    at org.scalatra.ScalatraBase$$anonfun$handle$1.apply(ScalatraBase.scala:123)
    at org.scalatra.ScalatraBase$$anonfun$handle$1.apply(ScalatraBase.scala:123)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
    at org.scalatra.DynamicScope$class.withResponse(DynamicScope.scala:78)
    at org.apache.livy.server.JsonServlet.withResponse(JsonServlet.scala:39)
    at org.scalatra.DynamicScope$$anonfun$withRequestResponse$1.apply(DynamicScope.scala:58)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
    at org.scalatra.DynamicScope$class.withRequest(DynamicScope.scala:69)
    at org.apache.livy.server.JsonServlet.withRequest(JsonServlet.scala:39)
    at org.scalatra.DynamicScope$class.withRequestResponse(DynamicScope.scala:57)
    at org.apache.livy.server.JsonServlet.withRequestResponse(JsonServlet.scala:39)
    at org.scalatra.ScalatraBase$class.handle(ScalatraBase.scala:122)
    at org.apache.livy.server.JsonServlet.org$scalatra$servlet$ServletBase$$super$handle(JsonServlet.scala:39)
    at org.scalatra.servlet.ServletBase$class.handle(ServletBase.scala:53)
    at org.apache.livy.server.SessionServlet.org$scalatra$MethodOverride$$super$handle(SessionServlet.scala:42)
    at org.scalatra.MethodOverride$class.handle(MethodOverride.scala:35)
–
    at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
    at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
    at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
    at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
    at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
    at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
    at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
    at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
    at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
    at java.lang.Thread.run(Thread.java:748)