Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-4890

stress crash: Coordinator race between TearDown() and GetNext() (crash dereferencing executor_)

    XMLWordPrintableJSON

Details

    Description

      The stress test had several successful runs (including with COMPUTE STATS statements) over the weekend at this commit:

      IMPALA-4876: Remove _test suffix from test names
      

      ... but one of the runs crashed here:

      0  0x00000039c6432625 in raise () from /lib64/libc.so.6
      #1  0x00000039c6433e05 in abort () from /lib64/libc.so.6
      #2  0x00007fb4c8c80a55 in os::abort(bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #3  0x00007fb4c8e00f87 in VMError::report_and_die() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #4  0x00007fb4c8c8596f in JVM_handle_linux_signal () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #5  <signal handler called>
      #6  0x00000000019aaec2 in boost::scoped_ptr<impala::RuntimeState>::get (this=0x3d8)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/smart_ptr/scoped_ptr.hpp:105
      #7  0x00000000019a9598 in impala::PlanFragmentExecutor::runtime_state (this=0x2b0)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.h:140
      #8  0x00000000019a4877 in impala::Coordinator::runtime_state (this=0x7fa7e9eb9b00)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/coordinator.cc:1557
      #9  0x000000000199f596 in impala::Coordinator::GetNext (this=0x7fa7e9eb9b00, results=0x7fa193cef840, max_rows=1024, eos=0x7fa13e9b0f49)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/coordinator.cc:1110
      #10 0x00000000014e7bea in impala::ImpalaServer::QueryExecState::FetchRowsInternal (this=0x7fa13e9b0400, max_rows=1024, fetched_rows=0x7fa193cef840)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/query-exec-state.cc:766
      #11 0x00000000014e7067 in impala::ImpalaServer::QueryExecState::FetchRows (this=0x7fa13e9b0400, max_rows=1024, fetched_rows=0x7fa193cef840)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/query-exec-state.cc:672
      #12 0x00000000014c2cf0 in impala::ImpalaServer::FetchInternal (this=0xe0fd000, query_id=..., fetch_size=1024, fetch_first=false, fetch_results=0x2adb7dd0)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/impala-hs2-server.cc:219
      #13 0x00000000014cc691 in impala::ImpalaServer::FetchResults (this=0xe0fd000, return_val=..., request=...)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/impala-hs2-server.cc:775
      #14 0x000000000151a971 in impala::ChildQuery::ExecAndFetch (this=0x2adb7d40)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/child-query.cc:84
      #15 0x000000000151fe15 in impala::ChildQueryExecutor::ExecChildQueries (this=0x7faabce103c0)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/service/child-query.cc:169
      #16 0x00000000015220c7 in boost::_mfi::mf0<void, impala::ChildQueryExecutor>::operator() (this=0x7fabf5964c48, p=0x7faabce103c0)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/bind/mem_fn_template.hpp:49
      #17 0x0000000001521e10 in boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> >::operator()<boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list0> (this=0x7fabf5964c58, f=..., a=...)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/bind/bind.hpp:253
      #18 0x0000000001521c4b in boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >::operator() (this=0x7fabf5964c48)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/bind/bind_template.hpp:20
      #19 0x0000000001521a9a in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >, void>::invoke (function_obj_ptr=...)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/function/function_template.hpp:153
      #20 0x000000000133bb90 in boost::function0<void>::operator() (this=0x7fabf5964c40)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/function/function_template.hpp:767
      #21 0x00000000015df1fb in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7fb421855a50)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/util/thread.cc:317
      #22 0x00000000015e61d4 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7fadc069d3c0,
          f=@0x7fadc069d3b8, a=...) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/bind/bind.hpp:457
      #23 0x00000000015e6117 in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7fadc069d3b8)
          at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/bind/bind_template.hpp:20
      #24 0x00000000015e6072 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (
          this=0x7fadc069d200) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0-p1/include/boost/thread/detail/thread.hpp:116
      #25 0x0000000001a4783a in thread_proxy ()
      #26 0x00000039c68079d1 in start_thread () from /lib64/libpthread.so.0
      #27 0x00000039c64e88fd in clone () from /lib64/libc.so.6
      

      Attachments

        Issue Links

          Activity

            People

              marcelk Marcel Kinard
              mikeb Michael Brown
              Votes:
              0 Vote for this issue
              Watchers:
              8 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: