Error when upgrading from Solr 8.0.0 to 8.5.2

I am trying to upgrade our tech stack from 8.0.0 to 8.5.2. The cofiguration work until the first time the ersver is rebooted. Then solr stops working with below error. The exact same config works with solr 8.0.0 without issues.

Extract  from solr.log:

021-02-15 19:04:52.922 INFO (main) [ ] o.a.s.c.ZkContainer Zookeeper client=10.201.52.56:2181,10.201.52.169:2181,10.201.52.136:2181
2021-02-15 19:04:52.940 INFO (main) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
2021-02-15 19:04:52.949 INFO (zkConnectionManagerCallback-9-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
2021-02-15 19:04:52.949 INFO (main) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
2021-02-15 19:04:53.047 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter Could not start Solr. Check solr/home property and the logs
2021-02-15 19:04:53.077 ERROR (main) [ ] o.a.s.c.SolrCore null:org.noggit.JSONParser$ParseException: Unexpected comma: char=,,position=324 AFTER='QuEFwT7s= nlE/oRd/BIthqBP8UY1yFiu6Betb7xDoyTPFD3AxaUo="}} },' BEFORE=' "authorization":

{ "class":"solr.R' at org.noggit.JSONParser.err(JSONParser.java:452) at org.noggit.JSONParser.next(JSONParser.java:1013) at org.noggit.JSONParser.nextEvent(JSONParser.java:1073) at org.noggit.ObjectBuilder.checkEOF(ObjectBuilder.java:52) at org.noggit.ObjectBuilder.getValStrict(ObjectBuilder.java:76) at org.apache.solr.common.util.Utils.fromJSON(Utils.java:270) at org.apache.solr.common.util.Utils.fromJSON(Utils.java:256) at org.apache.solr.common.cloud.ZkStateReader.getSecurityProps(ZkStateReader.java:1274) at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:527) at org.apache.solr.cloud.ZkController.init(ZkController.java:902) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:472) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:115) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:663) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:134) at org.eclipse.jetty.servlet.ServletHandler.lambda$initialize$0(ServletHandler.java:751) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:658) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:744) at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:360) at org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1445) at org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1409) at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:822) at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:275) at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:524) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(StandardStarter.java:46) at org.eclipse.jetty.deploy.AppLifeCycle.runBindings(AppLifeCycle.java:188) at org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(DeploymentManager.java:513) at org.eclipse.jetty.deploy.DeploymentManager.addApp(DeploymentManager.java:154) at org.eclipse.jetty.deploy.providers.ScanningAppProvider.fileAdded(ScanningAppProvider.java:173) at org.eclipse.jetty.deploy.providers.WebAppProvider.fileAdded(WebAppProvider.java:447) at org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(ScanningAppProvider.java:66) at org.eclipse.jetty.util.Scanner.reportAddition(Scanner.java:784) at org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:753) at org.eclipse.jetty.util.Scanner.scan(Scanner.java:641) at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:540) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.deploy.providers.ScanningAppProvider.doStart(ScanningAppProvider.java:146) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(DeploymentManager.java:599) at org.eclipse.jetty.deploy.DeploymentManager.doStart(DeploymentManager.java:249) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.server.Server.start(Server.java:407) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:100) at org.eclipse.jetty.server.Server.doStart(Server.java:371) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.xml.XmlConfiguration.lambda$main$0(XmlConfiguration.java:1888) at java.base/java.security.AccessController.doPrivileged(Native Method) at org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1837) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.eclipse.jetty.start.Main.invokeMain(Main.java:218) at org.eclipse.jetty.start.Main.start(Main.java:491) at org.eclipse.jetty.start.Main.main(Main.java:77) 2021-02-15 19:04:53.108 INFO (main) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@4628b1d3\{/solr,file:///opt/solr-8.5.2/server/solr-webapp/webapp/,AVAILABLE} {/opt/solr-8.5.2/server/solr-webapp/webapp}

2021-02-15 19:04:53.118 INFO (main) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d24ffa1{HTTP/1.1,[http/1.1, h2c]}

{0.0.0.0:8983}

2021-02-15 19:04:53.118 INFO (main) [ ] o.e.j.s.Server Started @7244ms
2021-02-15 19:05:28.284 ERROR (qtp1481594086-20) [ ] o.a.s.s.SolrDispatchFilter Error processing the request. CoreContainer is either not initialized or shutting down.
2021-02-15 19:05:28.289 WARN (qtp1481594086-20) [ ] o.e.j.s.HttpChannel /solr/admin/info/system => javax.servlet.ServletException: javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.

 

when running sudo service solr status

Found 1 Solr nodes:

Solr process 1891 running on port 8983

ERROR: Failed to get system information from http://localhost:8983/solr due to: org.apache.solr.common.SolrException: Parse error : <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</title>
</head>
<body><h2>HTTP ERROR 404 javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</h2>
<table>
<tr><th>URI:</th><td>/solr/admin/info/system</td></tr>
<tr><th>STATUS:</th><td>404</td></tr>
<tr><th>MESSAGE:</th><td>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</td></tr>
<tr><th>SERVLET:</th><td>default</td></tr>
<tr><th>CAUSED BY:</th><td>javax.servlet.ServletException: javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</td></tr>
<tr><th>CAUSED BY:</th><td>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</td></tr>
</table>
<h3>Caused by:</h3><pre>javax.servlet.ServletException: javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:162)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:371)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:352)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
at org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
... 14 more
</pre>

</body>
</html>