Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.10.0
-
ghx-label-3
Description
The preaggregation can crash if it can't get enough memory for its initial hash tables. The bug is that the Suballocator must be able to allocate at least one buffer of the default size to allocate any memory, but the calculation in the planner only reserves 512kb of memory.
F0828 15:54:41.898521 56405 partitioned-aggregation-node.cc:1128] Check failed: !is_streaming_preagg_ Preagg reserves enough memory for hash tables #0 0x0000003299a32625 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x0000003299a33e05 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x0000000002e7fc04 in google::DumpStackTraceAndExit() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #3 0x0000000002e7667d in google::LogMessage::Fail() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #4 0x0000000002e77f22 in google::LogMessage::SendToLog() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #5 0x0000000002e76057 in google::LogMessage::Flush() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #6 0x0000000002e7961e in google::LogMessageFatal::~LogMessageFatal() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #7 0x0000000001bef4a8 in impala::PartitionedAggregationNode::CreateHashPartitions (this=0x7f5b7b97e000, level=0, single_partition_idx=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:1128 got_memory = false partition = 0x7f602ce48850 i = 0 num_partitions_created = 16 #8 0x0000000001be6a9e in impala::PartitionedAggregationNode::Open (this=0x7f5b7b97e000, state=0x7f4a3adf5c00) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:290 __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f5a1d9c02f0} SCOPED_TIMER1 = {sw_ = {start_ = 2679682494289161, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5a8d3ecc88, is_cancelled_ = 0x0} batch = {static AT_CAPACITY_MEM_USAGE = 8388608, static FIXED_LEN_BUFFER_LIMIT = 4194304, num_rows_ = 228113440, capacity_ = 32605, flush_ = 496763888, needs_deep_copy_ = 90, num_tuples_per_row_ = -840859936, tuple_ptrs_size_ = 32607, tuple_ptrs_ = 0x7f5d2fd96cd8, auxiliary_mem_usage_ = 140025020547552, tuple_data_pool_ = { static LLVM_CLASS_NAME = 0x2fe5b94 "class.impala::MemPool", static DEFAULT_ALIGNMENT = 8, static INITIAL_CHUNK_SIZE = 4096, static MAX_CHUNK_SIZE = 1048576, static zero_length_region_ = 1722447803, current_chunk_idx_ = -840859936, next_chunk_size_ = 32607, total_allocated_bytes_ = 140025020547568, peak_allocated_bytes_ = 27193507, total_reserved_bytes_ = 1, chunks_ = {<std::_Vector_base<impala::MemPool::ChunkInfo, std::allocator<impala::MemPool::ChunkInfo> >> = {_M_impl = {<std::allocator<impala::MemPool::ChunkInfo>> = {<__gnu_cxx::new_allocator<impala::MemPool::ChunkInfo>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f5a1d9c03f0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5d2fd96cd8}}, <No data fields>}, mem_tracker_ = 0x7f5cd932ecb0}, row_desc_ = 0x8, mem_tracker_ = 0x7f5cd932ecb0, io_buffers_ = {<std::_Vector_base<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> >, std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > > >> = { _M_impl = {<std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<__gnu_cxx::new_allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x15c86d0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5a1d9c0780}}, <No data fields>}, buffers_ = {<std::_Vector_base<impala::RowBatch::BufferInfo, std::allocator<impala::RowBatch::BufferInfo> >> = { _M_impl = {<std::allocator<impala::RowBatch::BufferInfo>> = {<__gnu_cxx::new_allocator<impala::RowBatch::BufferInfo>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f5cd932ecb0, _M_finish = 0x7f5fcde17ee8, _M_end_of_storage = 0x0}}, <No data fields>}, compression_scratch_ = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5cd932ecb0 "\330\376A\323Z\177"}}} eos = false #9 0x00000000017f16a3 in impala::FragmentInstanceState::Open (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:256 __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x3008ab5} SCOPED_TIMER9 = {sw_ = {start_ = 2679682494285694, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5f0222fa80, is_cancelled_ = 0x0} SCOPED_TIMER6 = {sw_ = {start_ = 2679678789834010, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f4a3adf5f18, is_cancelled_ = 0x0} SCOPED_TIMER7 = {sw_ = {start_ = 2679678789843186, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f55634f4900, is_cancelled_ = 0x0} SCOPED_THREAD_COUNTER_MEASUREMENT8 = {stop_ = false, usage_base_ = {ru_utime = {tv_sec = 0, tv_usec = 102984}, ru_stime = {tv_sec = 0, tv_usec = 18997}, ru_maxrss = 171863496, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 407, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 24, ru_msgsnd = 0, ru_msgrcv = 0, ru_nsignals = 0, ru_nvcsw = 159, ru_nivcsw = 145}, sw_ = {start_ = 2679678789843580, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counters_ = 0x7f59c3505200} #10 0x00000000017ef1cf in impala::FragmentInstanceState::Exec (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:80 status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0} #11 0x00000000017d9bda in impala::QueryState::ExecFInstance (this=0x7f57d1299600, fis=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:351 status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0} #12 0x00000000017d870a in impala::QueryState::<lambda()>::operator()(void) const (__closure=0x7f5a1d9c0bc8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:319 fis = 0x7f57f3f5af80 this = 0x7f57d1299600 #13 0x00000000017da88b in boost::detail::function::void_function_obj_invoker0<impala::QueryState::StartFInstances()::<lambda()>, void>::invoke(boost::detail::function::function_buffer &) (function_obj_ptr=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153 f = 0x7f5a1d9c0bc8 #14 0x000000000170b722 in boost::function0<void>::operator() (this=0x7f5a1d9c0bc0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767 No locals. #15 0x00000000019fc6cd in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f61361b8be0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:329 system_tid = 56730 category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6910678 "fragment-execution"}} thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x695e680, _M_refcount = {_M_pi = 0x696c1a0}}, <No data fields>} name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f568a417038 "exec-finstance (finst:5a42932fcf62becc:f7747b5800000038)"}} #16 0x0000000001a05008 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f5f67979dc0, f=@0x7f5f67979db8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457 No locals. #17 0x0000000001a04f4b in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f5f67979db8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20 a = {<No data fields>} #18 0x0000000001a04f0e in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f5f67979c00) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116 No locals. #19 0x00000000020889aa in thread_proxy () DEBUG_MODE = true kint32min = -2147483648 kint16min = -32768 kint8max = 127 '\177' kMaxFprint = 18446744073709551615 PATH_SEPARATOR = 47 '/' kuint32max = 4294967295 kuint8max = 255 '\377' kint32max = 2147483647 kint64min = -9223372036854775808 kIllegalFprint = 0 kuint16max = 65535 kint16max = 32767 kuint64max = 18446744073709551615 kint64max = 9223372036854775807 kint8min = -128 '\200' Bits::num_bits = 0x3fbb740 "" #20 0x0000003299e07aa1 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #21 0x0000003299ae893d in clone () from /lib64/libc.so.6 No symbol table info available.
Repro:
set debug_action=-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0; select distinct o_orderdate, o_custkey, o_comment from tpch_parquet.orders;