Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.5.0
-
None
Description
Encountered the following crash when investigating IMPALA-3068.
The problem is that DataSink objects MemTrackers may have been destroyed before but they are still connected to the tree of MemTrackers.
11:43:30 #5 0x00007fa55eef8348 in __cxa_throw () at ../../../../gcc-4.9.2/libstdc++-v3/libsupc++/eh_throw.cc:87 11:43:30 #6 0x0000000001080e92 in boost::throw_exception<boost::lock_error> (e=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/throw_exception.hpp:69 11:43:30 #7 0x0000000001078812 in boost::mutex::lock (this=0xe672208) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/thread/pthread/mutex.hpp:119 11:43:30 #8 0x0000000001081ef0 in boost::lock_guard<boost::mutex>::lock_guard (this=0x7fa4b199b270, m_=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/thread/lock_guard.hpp:38 11:43:30 #9 0x0000000001236ec7 in impala::MemTracker::LogUsage (this=0xe672160, prefix=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/mem-tracker.cc:275 11:43:30 #10 0x000000000123709f in impala::MemTracker::LogUsage (prefix=..., trackers=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/mem-tracker.cc:285 11:43:30 #11 0x0000000001236ee8 in impala::MemTracker::LogUsage (this=0x1670f8c0, prefix=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/mem-tracker.cc:276 11:43:30 #12 0x000000000123709f in impala::MemTracker::LogUsage (prefix=..., trackers=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/mem-tracker.cc:285 11:43:30 #13 0x0000000001236ee8 in impala::MemTracker::LogUsage (this=0xa242c60, prefix=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/mem-tracker.cc:276 11:43:30 #14 0x0000000001294243 in impala::RuntimeState::SetMemLimitExceeded (this=0xf978400, tracker=0x0, failed_allocation_size=0, msg=0x0) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/runtime-state.cc:269 11:43:30 #15 0x00000000012946de in impala::RuntimeState::CheckQueryState (this=0xf978400) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/runtime/runtime-state.cc:286 11:43:30 #16 0x00000000016552ed in impala::HdfsScanner::CommitRows (this=0x7e1e9c0, num_rows=1) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/exec/hdfs-scanner.cc:182 11:43:30 #17 0x0000000001573830 in impala::HdfsSequenceScanner::ProcessRange (this=0x7e1e9c0) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/exec/hdfs-sequence-scanner.cc:327 11:43:30 #18 0x000000000163d5e9 in impala::BaseSequenceScanner::ProcessSplit (this=0x7e1e9c0) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/exec/base-sequence-scanner.cc:144 11:43:30 #19 0x000000000154aad7 in impala::HdfsScanNode::ProcessSplit (this=0x1d6ca400, filter_ctxs=..., scan_range=0x16861ce0) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/exec/hdfs-scan-node.cc:1163 11:43:30 #20 0x0000000001549cd9 in impala::HdfsScanNode::ScannerThread (this=0x1d6ca400) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/exec/hdfs-scan-node.cc:1043 11:43:30 #21 0x0000000001566db1 in boost::_mfi::mf0<void, impala::HdfsScanNode>::operator() (this=0x7fa4b199ccc8, p=0x1d6ca400) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:49 11:43:30 #22 0x0000000001565f34 in boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> >::operator()<boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list0> (this=0x7fa4b199ccd8, f=..., a=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:253 11:43:30 #23 0x00000000015649e3 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> > >::operator() (this=0x7fa4b199ccc8) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20 11:43:30 #24 0x00000000015627b0 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> > >, void>::invoke (function_obj_ptr=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:153 11:43:30 #25 0x000000000121bce6 in boost::function0<void>::operator() (this=0x7fa4b199ccc0) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:767 11:43:30 #26 0x00000000014c9111 in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7fa4afb98380) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/be/src/util/thread.cc:316 11:43:30 #27 0x00000000014d078c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0xb2ce3c0, f=@0xb2ce3b8, a=...) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:457 11:43:30 #28 0x00000000014d06cf in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0xb2ce3b8) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20 11:43:30 #29 0x00000000014d0692 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0xb2ce200) at /data/jenkins/workspace/impala-master-64bit-PRIVATE-fast/repos/Impala/toolchain/boost-1.57.0/include/boost/thread/detail/thread.hpp:116 11:43:30 #30 0x000000000189f70a in thread_proxy () 11:43:30 #31 0x000000353c6079d1 in start_thread () from /lib64/libpthread.so.0 11:43:30 #32 0x000000353c2e88fd in clone () from /lib64/libc.so.6