Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-5855

Preaggregation crashes - unable to initialise hash table

    XMLWordPrintableJSON

Details

    Description

      The preaggregation can crash if it can't get enough memory for its initial hash tables. The bug is that the Suballocator must be able to allocate at least one buffer of the default size to allocate any memory, but the calculation in the planner only reserves 512kb of memory.

      F0828 15:54:41.898521 56405 partitioned-aggregation-node.cc:1128] Check failed: !is_streaming_preagg_ Preagg reserves enough memory for hash tables
      
      #0  0x0000003299a32625 in raise () from /lib64/libc.so.6
      No symbol table info available.
      #1  0x0000003299a33e05 in abort () from /lib64/libc.so.6
      No symbol table info available.
      #2  0x0000000002e7fc04 in google::DumpStackTraceAndExit() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #3  0x0000000002e7667d in google::LogMessage::Fail() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #4  0x0000000002e77f22 in google::LogMessage::SendToLog() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #5  0x0000000002e76057 in google::LogMessage::Flush() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #6  0x0000000002e7961e in google::LogMessageFatal::~LogMessageFatal() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #7  0x0000000001bef4a8 in impala::PartitionedAggregationNode::CreateHashPartitions (this=0x7f5b7b97e000, level=0, single_partition_idx=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:1128
              got_memory = false
              partition = 0x7f602ce48850
              i = 0
              num_partitions_created = 16
      #8  0x0000000001be6a9e in impala::PartitionedAggregationNode::Open (this=0x7f5b7b97e000, state=0x7f4a3adf5c00) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:290
              __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f5a1d9c02f0}
              SCOPED_TIMER1 = {sw_ = {start_ = 2679682494289161, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5a8d3ecc88, is_cancelled_ = 0x0}
              batch = {static AT_CAPACITY_MEM_USAGE = 8388608, static FIXED_LEN_BUFFER_LIMIT = 4194304, num_rows_ = 228113440, capacity_ = 32605, flush_ = 496763888, needs_deep_copy_ = 90, num_tuples_per_row_ = -840859936, tuple_ptrs_size_ = 32607, tuple_ptrs_ = 0x7f5d2fd96cd8, auxiliary_mem_usage_ = 140025020547552, tuple_data_pool_ = {
                  static LLVM_CLASS_NAME = 0x2fe5b94 "class.impala::MemPool", static DEFAULT_ALIGNMENT = 8, static INITIAL_CHUNK_SIZE = 4096, static MAX_CHUNK_SIZE = 1048576, static zero_length_region_ = 1722447803, current_chunk_idx_ = -840859936, next_chunk_size_ = 32607, total_allocated_bytes_ = 140025020547568, 
                  peak_allocated_bytes_ = 27193507, total_reserved_bytes_ = 1, chunks_ = {<std::_Vector_base<impala::MemPool::ChunkInfo, std::allocator<impala::MemPool::ChunkInfo> >> = {_M_impl = {<std::allocator<impala::MemPool::ChunkInfo>> = {<__gnu_cxx::new_allocator<impala::MemPool::ChunkInfo>> = {<No data fields>}, <No data fields>}, 
                        _M_start = 0x7f5a1d9c03f0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5d2fd96cd8}}, <No data fields>}, mem_tracker_ = 0x7f5cd932ecb0}, row_desc_ = 0x8, mem_tracker_ = 0x7f5cd932ecb0, 
                io_buffers_ = {<std::_Vector_base<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> >, std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > > >> = {
                    _M_impl = {<std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<__gnu_cxx::new_allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x15c86d0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5a1d9c0780}}, <No data fields>}, buffers_ = {<std::_Vector_base<impala::RowBatch::BufferInfo, std::allocator<impala::RowBatch::BufferInfo> >> = {
                    _M_impl = {<std::allocator<impala::RowBatch::BufferInfo>> = {<__gnu_cxx::new_allocator<impala::RowBatch::BufferInfo>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f5cd932ecb0, _M_finish = 0x7f5fcde17ee8, _M_end_of_storage = 0x0}}, <No data fields>}, compression_scratch_ = {static npos = 18446744073709551615, 
                  _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5cd932ecb0 "\330\376A\323Z\177"}}}
              eos = false
      #9  0x00000000017f16a3 in impala::FragmentInstanceState::Open (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:256
              __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x3008ab5}
              SCOPED_TIMER9 = {sw_ = {start_ = 2679682494285694, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5f0222fa80, is_cancelled_ = 0x0}
              SCOPED_TIMER6 = {sw_ = {start_ = 2679678789834010, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f4a3adf5f18, is_cancelled_ = 0x0}
              SCOPED_TIMER7 = {sw_ = {start_ = 2679678789843186, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f55634f4900, is_cancelled_ = 0x0}
              SCOPED_THREAD_COUNTER_MEASUREMENT8 = {stop_ = false, usage_base_ = {ru_utime = {tv_sec = 0, tv_usec = 102984}, ru_stime = {tv_sec = 0, tv_usec = 18997}, ru_maxrss = 171863496, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 407, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 24, ru_msgsnd = 0, ru_msgrcv = 0, 
                  ru_nsignals = 0, ru_nvcsw = 159, ru_nivcsw = 145}, sw_ = {start_ = 2679678789843580, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counters_ = 0x7f59c3505200}
      #10 0x00000000017ef1cf in impala::FragmentInstanceState::Exec (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:80
              status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
      #11 0x00000000017d9bda in impala::QueryState::ExecFInstance (this=0x7f57d1299600, fis=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:351
              status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
      #12 0x00000000017d870a in impala::QueryState::<lambda()>::operator()(void) const (__closure=0x7f5a1d9c0bc8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:319
              fis = 0x7f57f3f5af80
              this = 0x7f57d1299600
      #13 0x00000000017da88b in boost::detail::function::void_function_obj_invoker0<impala::QueryState::StartFInstances()::<lambda()>, void>::invoke(boost::detail::function::function_buffer &) (function_obj_ptr=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
              f = 0x7f5a1d9c0bc8
      #14 0x000000000170b722 in boost::function0<void>::operator() (this=0x7f5a1d9c0bc0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
      No locals.
      #15 0x00000000019fc6cd in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f61361b8be0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:329
              system_tid = 56730
              category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6910678 "fragment-execution"}}
              thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x695e680, _M_refcount = {_M_pi = 0x696c1a0}}, <No data fields>}
              name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f568a417038 "exec-finstance (finst:5a42932fcf62becc:f7747b5800000038)"}}
      #16 0x0000000001a05008 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f5f67979dc0, f=@0x7f5f67979db8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
      No locals.
      #17 0x0000000001a04f4b in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f5f67979db8)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
              a = {<No data fields>}
      #18 0x0000000001a04f0e in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f5f67979c00)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
      No locals.
      #19 0x00000000020889aa in thread_proxy ()
              DEBUG_MODE = true
              kint32min = -2147483648
              kint16min = -32768
              kint8max = 127 '\177'
              kMaxFprint = 18446744073709551615
              PATH_SEPARATOR = 47 '/'
              kuint32max = 4294967295
              kuint8max = 255 '\377'
              kint32max = 2147483647
              kint64min = -9223372036854775808
              kIllegalFprint = 0
              kuint16max = 65535
              kint16max = 32767
              kuint64max = 18446744073709551615
              kint64max = 9223372036854775807
              kint8min = -128 '\200'
              Bits::num_bits = 0x3fbb740 ""
      #20 0x0000003299e07aa1 in start_thread () from /lib64/libpthread.so.0
      No symbol table info available.
      #21 0x0000003299ae893d in clone () from /lib64/libc.so.6
      No symbol table info available.
      

      Repro:

      set debug_action=-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0;
      select distinct o_orderdate, o_custkey, o_comment
      from tpch_parquet.orders;
      

      Attachments

        Activity

          People

            tarmstrong Tim Armstrong
            mmulder Matthew Mulder
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: