Description
I was testing a crazy surround query and was hitting OOMs easily with the query. However I saw that the OOM killer wasn't triggered. Here is the stack trace of the error on solr 5.4:
WARN - 2016-01-12 18:37:03.920; [ x:techproducts] org.eclipse.jetty.util.thread.QueuedThreadPool$3;
java.lang.OutOfMemoryError: Java heap space
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.addConditionWaiter(AbstractQueuedSynchronizer.java:1855)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2068)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)
at java.lang.Thread.run(Thread.java:745)
ERROR - 2016-01-12 18:37:03.922; [ x:techproducts] org.apache.solr.common.SolrException; null:java.lang.RuntimeException: java.lang.OutOfMemoryError: Java heap space
at org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:611)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:472)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:222)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:181)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.newTermState(Lucene50PostingsReader.java:149)
at org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:100)
at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekCeil(SegmentTermsEnum.java:728)
at org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.seekCeil(FilterLeafReader.java:185)
at org.apache.lucene.index.TermsEnum.seekExact(TermsEnum.java:74)
at org.apache.lucene.index.TermContext.build(TermContext.java:94)
at org.apache.lucene.search.spans.SpanTermQuery.createWeight(SpanTermQuery.java:72)
at org.apache.lucene.search.spans.SpanOrQuery.createWeight(SpanOrQuery.java:132)
at org.apache.lucene.search.spans.SpanNearQuery.createWeight(SpanNearQuery.java:192)
at org.apache.lucene.search.spans.SpanNearQuery.createWeight(SpanNearQuery.java:42)
at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:904)
at org.apache.lucene.search.IndexSearcher.createNormalizedWeight(IndexSearcher.java:887)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
at org.apache.solr.search.SolrIndexSearcher.buildAndRunCollectorChain(SolrIndexSearcher.java:202)
at org.apache.solr.search.SolrIndexSearcher.getDocListNC(SolrIndexSearcher.java:1672)
at org.apache.solr.search.SolrIndexSearcher.getDocListC(SolrIndexSearcher.java:1491)
at org.apache.solr.search.SolrIndexSearcher.search(SolrIndexSearcher.java:557)
at org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:525)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:273)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:156)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2073)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:658)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:457)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:222)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:181)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
So HttpSolrCall#sendError needs to throw an OOM exception instead of swallowing it and sending back and error in the response.
Once I made HttpSolrCall#sendError throw an exception on OOM I still hit the following stack strace
WARN - 2016-01-13 07:43:39.449; [ x:techproducts] org.eclipse.jetty.servlet.ServletHandler; Error for /solr/techproducts%2Fselect
java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader$BlockPostingsEnum.<init>(Lucene50PostingsReader.java:477)
at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.postings(Lucene50PostingsReader.java:220)
at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1002)
at org.apache.lucene.search.spans.SpanTermQuery$SpanTermWeight.getSpans(SpanTermQuery.java:119)
at org.apache.lucene.search.spans.SpanOrQuery$SpanOrWeight.getSpans(SpanOrQuery.java:160)
at org.apache.lucene.search.spans.SpanNearQuery$SpanNearWeight.getSpans(SpanNearQuery.java:213)
at org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:133)
at org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:38)
at org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)
at org.apache.solr.search.SolrIndexSearcher.buildAndRunCollectorChain(SolrIndexSearcher.java:240)
at org.apache.solr.search.SolrIndexSearcher.getDocListNC(SolrIndexSearcher.java:1766)
at org.apache.solr.search.SolrIndexSearcher.getDocListC(SolrIndexSearcher.java:1583)
at org.apache.solr.search.SolrIndexSearcher.search(SolrIndexSearcher.java:613)
at org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:524)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:273)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:156)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2062)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:653)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:457)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:226)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
So is Jetty not propagating the OOM Error back to the JVM?
Attachments
Attachments
Issue Links
- relates to
-
SOLR-4992 Solr queries don't propagate Java OutOfMemoryError back to the JVM
-
- Closed
-