Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-3142

High contention for spinlock in MemTracker::GcMemory()

    XMLWordPrintableJSON

Details

    Description

      Mostafa reported that Impala could get stuck spinning on a spinlock in GcMemory():

      It seems that the spinning is allocations which makes lots of requests to the MemTracker.
      __sched_yield has a CPI of 10 which is really high.

      Instructions Retired
      1 of 61: 32.4% (84688982810 of 261749989945)
      
      libc-2.12.so ! __sched_yield - [unknown source file]
      libc-2.12.so ! [Unknown stack frame(s)] - [unknown source file]
      impalad ! impala::MemTracker::LimitExceeded + 0x4f - mem-tracker.h:268
      impalad ! impala::MemTracker::AnyLimitExceeded + 0x15 - mem-tracker.h:255
      impalad ! impala::RuntimeState::CheckQueryState + 0x1b - runtime-state.cc:287
      impalad ! impala::HdfsScanner::CommitRows + 0xce - hdfs-scanner.cc:182
      impalad ! impala::HdfsParquetScanner::AssembleRows<(bool)1, (bool)0> + 0x252 - hdfs-parquet-scanner.cc:1657
      impalad ! impala::HdfsParquetScanner::ProcessSplit + 0x31f - hdfs-parquet-scanner.cc:1535
      impalad ! impala::HdfsScanNode::ProcessSplit + 0x365 - hdfs-scan-node.cc:1183
      impalad ! impala::HdfsScanNode::ScannerThread + 0xc8d - hdfs-scan-node.cc:1063
      impalad ! boost::function0<void>::operator() + 0x1a - function_template.hpp:767
      impalad ! impala::Thread::SuperviseThread + 0x20c - thread.cc:316
      impalad ! operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
      impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() - bind_template.hpp:20
      impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 - thread.hpp:116
      impalad ! thread_proxy + 0xd9 - [unknown source file]
      libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
      libc-2.12.so ! clone + 0x6c - [unknown source file]
      
      Instructions Retired
      2 of 61: 13.5% (35412538191 of 261749989945)
      
      libc-2.12.so ! __sched_yield - [unknown source file]
      libc-2.12.so ! [Unknown stack frame(s)] - [unknown source file]
      impalad ! impala::MemTracker::LimitExceeded + 0x4f - mem-tracker.h:268
      impalad ! impala::MemTracker::AnyLimitExceeded + 0x15 - mem-tracker.h:255
      impalad ! impala::RuntimeState::CheckQueryState + 0x1b - runtime-state.cc:287
      impalad ! impala::HdfsScanner::CommitRows + 0xce - hdfs-scanner.cc:182
      impalad ! impala::HdfsParquetScanner::AssembleRows<(bool)0, (bool)0> + 0x82e - hdfs-parquet-scanner.cc:1657
      impalad ! impala::HdfsParquetScanner::ProcessSplit + 0x4d2 - hdfs-parquet-scanner.cc:1535
      impalad ! impala::HdfsScanNode::ProcessSplit + 0x365 - hdfs-scan-node.cc:1183
      impalad ! impala::HdfsScanNode::ScannerThread + 0xc8d - hdfs-scan-node.cc:1063
      impalad ! boost::function0<void>::operator() + 0x1a - function_template.hpp:767
      impalad ! impala::Thread::SuperviseThread + 0x20c - thread.cc:316
      impalad ! operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
      impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() - bind_template.hpp:20
      impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 - thread.hpp:116
      impalad ! thread_proxy + 0xd9 - [unknown source file]
      libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
      libc-2.12.so ! clone + 0x6c - [unknown source file]
      
      Instructions Retired
      4 of 61: 6.0% (15807873279 of 261749989945)
      
      libc-2.12.so ! __sched_yield - [unknown source file]
      libc-2.12.so ! [Unknown stack frame(s)] - [unknown source file]
      impalad ! impala::MemTracker::TryConsume + 0x1c3 - mem-tracker.h:184
      impalad ! impala::MemPool::FindChunk + 0x140 - mem-pool.cc:129
      impalad ! Allocate<true> + 0x57 - mem-pool.h:215
      impalad ! impala::MemPool::TryAllocate - mem-pool.h:99
      impalad ! CollectionValueBuilder + 0x31 - collection-value-builder.h:37
      impalad ! impala::HdfsParquetScanner::CollectionColumnReader::ReadSlot - hdfs-parquet-scanner.cc:1359
      impalad ! impala::HdfsParquetScanner::CollectionColumnReader::ReadValue + 0x4c - hdfs-parquet-scanner.cc:1333
      impalad ! ReadRow<true> + 0x7d - hdfs-parquet-scanner.cc:1701
      impalad ! impala::HdfsParquetScanner::AssembleRows<(bool)1, (bool)1> + 0x100 - hdfs-parquet-scanner.cc:1610
      impalad ! impala::HdfsParquetScanner::CollectionColumnReader::ReadSlot + 0x10c - hdfs-parquet-scanner.cc:1363
      impalad ! impala::HdfsParquetScanner::CollectionColumnReader::ReadValue + 0x4c - hdfs-parquet-scanner.cc:1333
      impalad ! ReadRow<true> + 0x82 - hdfs-parquet-scanner.cc:1701
      impalad ! impala::HdfsParquetScanner::AssembleRows<(bool)1, (bool)0> + 0xf0 - hdfs-parquet-scanner.cc:1593
      impalad ! impala::HdfsParquetScanner::ProcessSplit + 0x31f - hdfs-parquet-scanner.cc:1535
      impalad ! impala::HdfsScanNode::ProcessSplit + 0x365 - hdfs-scan-node.cc:1183
      impalad ! impala::HdfsScanNode::ScannerThread + 0xc8d - hdfs-scan-node.cc:1063
      impalad ! boost::function0<void>::operator() + 0x1a - function_template.hpp:767
      impalad ! impala::Thread::SuperviseThread + 0x20c - thread.cc:316
      impalad ! operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
      impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() - bind_template.hpp:20
      impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 - thread.hpp:116
      impalad ! thread_proxy + 0xd9 - [unknown source file]
      libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
      libc-2.12.so ! clone + 0x6c - [unknown source file]
      
      Instructions Retired
      2 of 49: 4.5% (662091353 of 14656153166)
      
      vmlinux ! thread_return - [unknown source file]
      vmlinux ! sys_sched_yield + 0x54 - [unknown source file]
      vmlinux ! system_call_fastpath + 0x15 - [unknown source file]
      libc-2.12.so ! __sched_yield + 0x6 - [unknown source file]
      impalad ! impala::SpinLock::SlowAcquire + 0x40 - spinlock.cc:25
      impalad ! lock_guard + 0x4 - lock_guard.hpp:38
      impalad ! impala::MemTracker::GcMemory + 0x19 - mem-tracker.cc:301
      impalad ! impala::MemTracker::LimitExceeded + 0x60 - mem-tracker.h:268
      impalad ! impala::DiskIoMgr::GetNextRequestRange + 0x90 - disk-io-mgr.cc:786
      impalad ! impala::DiskIoMgr::WorkLoop + 0x41 - disk-io-mgr.cc:987
      impalad ! boost::function0<void>::operator() + 0x1a - function_template.hpp:767
      impalad ! impala::Thread::SuperviseThread + 0x20c - thread.cc:316
      impalad ! operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
      impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() - bind_template.hpp:20
      impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 - thread.hpp:116
      impalad ! thread_proxy + 0xd9 - [unknown source file]
      libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
      libc-2.12.so ! clone + 0x6c - [unknown source file]
      

      Attachments

        Activity

          People

            tarmstrong Tim Armstrong
            tarmstrong Tim Armstrong
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: