Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-1867

Crash: DCHECK: mem_tracker()->consumption() == 0 (8388608 vs. 0) Leaked memory.

    XMLWordPrintableJSON

Details

    Description

      There were 11 randomly generated queries executing concurrently on the 10 node CDH5 cluster on tpch300gb_parquet DB. mem_limit was not set.

      Stack Trace:

      #0  0x00000030770328e5 in raise () from /lib64/libc.so.6
      #0  0x00000030770328e5 in raise () from /lib64/libc.so.6
      #1  0x00000030770340c5 in abort () from /lib64/libc.so.6
      #2  0x0000000001f0de09 in google::DumpStackTraceAndExit () at src/utilities.cc:147
      #3  0x0000000001f0542d in google::LogMessage::Fail () at src/logging.cc:1315
      #4  0x0000000001f08fc7 in google::LogMessage::SendToLog (this=0x7f1c0f961c00) at src/logging.cc:1269
      #5  0x0000000001f079fa in google::LogMessage::Flush (this=0x7f1c0f961c00) at src/logging.cc:1138
      #6  0x0000000001f07ccd in google::LogMessageFatal::~LogMessageFatal (this=0x7f1c0f961c00, __in_chrg=<value optimized out>) at src/logging.cc:1836
      #7  0x0000000001471317 in impala::ExecNode::Close (this=0x7f2149e27800, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/exec-node.cc:176
      #8  0x00000000015235bb in impala::PartitionedAggregationNode::Close (this=0x7f2149e27800, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/partitioned-aggregation-node.cc:408
      #9  0x0000000001470fd5 in impala::ExecNode::Close (this=0x367324000, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/exec-node.cc:167
      #10 0x0000000001553674 in impala::SortNode::Close (this=0x367324000, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/sort-node.cc:122
      #11 0x0000000001470fd5 in impala::ExecNode::Close (this=0x7f258fbca380, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/exec-node.cc:167
      #12 0x000000000156cd9e in impala::AnalyticEvalNode::Close (this=0x7f258fbca380, state=0x1c093c00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/analytic-eval-node.cc:750
      #13 0x000000000143ed75 in impala::PlanFragmentExecutor::Close (this=0x31b25bd10) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.cc:570
      #14 0x00000000010966f6 in impala::FragmentMgr::FragmentExecState::Exec (this=0x31b25bb00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/service/fragment-exec-state.cc:50
      #15 0x000000000108db1c in impala::FragmentMgr::FragmentExecThread (this=0x76ed920, exec_state=0x31b25bb00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/service/fragment-mgr.cc:70
      #16 0x00000000010920f0 in boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>::operator() (this=0xc028fe0, p=0x76ed920, a1=0x31b25bb00) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/mem_fn_template.hpp:165
      #17 0x0000000001091ea9 in boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list0> (this=0xc028ff0, f=..., a=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind.hpp:313
      #18 0x0000000001091791 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >::operator() (this=0xc028fe0) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind_template.hpp:20
      #19 0x00000000010910df in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >, void>::invoke (function_obj_ptr=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/function/function_template.hpp:153
      #20 0x0000000000f21725 in boost::function0<void>::operator() (this=0x7f1c0f962cc0) at /opt/toolchain/boost-pic-1.55.0/include/boost/function/function_template.hpp:767
      #21 0x000000000117a090 in impala::Thread::SuperviseThread (name="exec-plan-fragment", category="impala-server", functor=..., thread_started=0x7f2583128b90) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/util/thread.cc:311
      #22 0x000000000118322c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::string &, const std::string &, impala::Thread::ThreadFunctor, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f258faeebc0, f=@0x7f258faeebb8, a=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind.hpp:457
      #23 0x0000000001183173 in boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f258faeebb8) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind_template.hpp:20
      #24 0x0000000001183132 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f258faeea00) at /opt/toolchain/boost-pic-1.55.0/include/boost/thread/detail/thread.hpp:117
      #25 0x00000000015e3793 in thread_proxy ()
      #26 0x0000003077407851 in start_thread () from /lib64/libpthread.so.0
      #27 0x00000030770e894d in clone () from /lib64/libc.so.6
      

      Additional information is located on e1118.halxg.cloudera.com in /data/4/IMPALA-1867 directory.

      Attachments

        1. second-tracker.31181.A15.65536.txt
          55 kB
          Ippokratis Pandis
        2. second-tracker.31181.A15.txt
          91 kB
          Ippokratis Pandis
        3. second-tracker.txt
          623 kB
          Ippokratis Pandis

        Issue Links

          Activity

            People

              dhecht Daniel Hecht
              tarasbob Taras Bobrovytsky
              Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: