Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.8.0, Impala 2.9.0, Impala 2.10.0, Impala 2.11.0, Impala 3.0, Impala 2.12.0
-
ghx-label-3
Description
I think this is related to the ExchangeNode's use of buffers.
Repro: increase the mem_limit for this test:
tarmstrong@tarmstrong-ubuntu:~/Impala$ git diff diff --git a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test index a6b3cae..7e8e862 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test +++ b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test @@ -1,7 +1,7 @@ ==== ---- QUERY # IMPALA-2207: Analytic eval node feeding into build side of nested loop join. -set mem_limit=200m; +set mem_limit=220m; select straight_join * from (values(1 id), (2), (3)) v1, (select *, count(*) over() from tpch.lineitem where l_orderkey < 100000) v2 order by id, l_orderkey, l_partkey, l_suppkey, l_linenumber
Loop the test:
while impala-py.test tests/query_test/test_join_queries.py -k 'test_single_node_nested_loop_joins_exhaustive' --workload_exploration_strategy=functional-query:exhaustive; do :; done
Boom:
(gdb) bt #0 0x00007fcffc535428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54 #1 0x00007fcffc53702a in __GI_abort () at abort.c:89 #2 0x00007fcfff479069 in os::abort(bool) (dump_core=<optimized out>) at /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os/linux/vm/os_linux.cpp:1509 #3 0x00007fcfff62c997 in VMError::report_and_die() (this=this@entry=0x7fcf8b201f50) at /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/share/vm/utilities/vmError.cpp:1060 #4 0x00007fcfff48254f in JVM_handle_linux_signal(int, siginfo_t*, void*, int) (sig=sig@entry=11, info=info@entry=0x7fcf8b2021f0, ucVoid=ucVoid@entry=0x7fcf8b2020c0, abort_if_unrecognized=abort_if_unrecognized@entry=1) at /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os_cpu/linux_x86/vm/os_linux_x86.cpp:541 #5 0x00007fcfff4761a8 in signalHandler(int, siginfo_t*, void*) (sig=11, info=0x7fcf8b2021f0, uc=0x7fcf8b2020c0) at /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os/linux/vm/os_linux.cpp:4346 #6 0x00007fcffc8db390 in <signal handler called> () at /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007fcffc8d3d44 in __GI___pthread_mutex_lock (mutex=0xd8) at ../nptl/pthread_mutex_lock.c:67 #8 0x0000000000d7f680 in impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*, impala::BufferPool::BufferHandle*) (m=0xd8) at toolchain/boost-1.57.0-p3/include/boost/thread/pthread/mutex.hpp:62 #9 0x0000000000d7f680 in impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*, impala::BufferPool::BufferHandle*) (this=0xd8) at toolchain/boost-1.57.0-p3/include/boost/thread/pthread/mutex.hpp:116 #10 0x0000000000d7f680 in impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*, impala::BufferPool::BufferHandle*) (m_=..., this=<synthetic pointer>) at toolchain/boost-1.57.0-p3/include/boost/thread/lock_guard.hpp:38 #11 0x0000000000d7f680 in impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*, impala::BufferPool::BufferHandle*) (len=2097152, this=0x0) at be/src/runtime/bufferpool/buffer-pool-internal.h:262 #12 0x0000000000d7f680 in impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*, impala::BufferPool::BufferHandle*) (this=<optimized out>, client=<optimized out>, handle=handle@entry=0xb971418) at be/src/runtime/bufferpool/buffer-pool.cc:254 #13 0x0000000000b66fc2 in impala::RowBatch::FreeBuffers() (this=this@entry=0x1bfbfea0) at be/src/runtime/row-batch.cc:425 #14 0x0000000000b67002 in impala::RowBatch::~RowBatch() (this=0x1bfbfea0, __in_chrg=<optimized out>) at be/src/runtime/row-batch.cc:220 #15 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=<optimized out>, __ptr=0x1bfbfea0) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:76 #16 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=0xa462628, __in_chrg=<optimized out>) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:236 #17 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (__pointer=0xa462628) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:93 #18 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (__last=0xa462658, __first=0xa462628) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:103 #19 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (__last=0xa462658, __first=0xa462000) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:126 #20 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (__last=0xa462658, __first=0xa462000) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:151 #21 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=<optimized out>, __pos=0xa462000) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_vector.h:1438 #22 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=0xf84ada8) at /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_vector.h:1212 #23 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=0xf84ad98) at be/src/exec/row-batch-cache.h:64 #24 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*) (this=0xf84ad20, state=0x111b8300) at be/src/exec/nested-loop-join-builder.cc:83 #25 0x0000000000e38050 in impala::NestedLoopJoinNode::Close(impala::RuntimeState*) (this=0x111b5900, state=0x111b8300) at be/src/exec/nested-loop-join-node.cc:141 #26 0x0000000000dc1041 in impala::ExecNode::Close(impala::RuntimeState*) (this=0x111b5400, state=0x111b8300) at be/src/exec/exec-node.cc:194 #27 0x0000000000baa02a in impala::FragmentInstanceState::Close() (this=this@entry=0x9102000) at be/src/runtime/fragment-instance-state.cc:310 #28 0x0000000000bad801 in impala::FragmentInstanceState::Exec() (this=this@entry=0x9102000) at be/src/runtime/fragment-instance-state.cc:95 #29 0x0000000000b9c00a in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) (this=0x9129800, fis=0x9102000) at be/src/runtime/query-state.cc:388 #30 0x0000000000d5be8f in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*) (this=0x7fcf8b202d50) at toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767 #31 0x0000000000d5be8f in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*) (name=..., category=..., functor=..., parent_thread_info=0x7fcf8c203ac0, thread_started=0x7fcf8c202fa0) at be/src/util/thread.cc:356 #32 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long>*> > > >::run() (f=@0x91023b8: 0xd5bb90 <impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*)>, a=<synthetic pointer>, this=0x91023c0) at toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:525 #33 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x91023b8) at toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20 #34 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x9102200) at toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116 #35 0x00000000012d4e7a in thread_proxy () #36 0x00007fcffc8d16ba in start_thread (arg=0x7fcf8b203700) at pthread_create.c:333 #37 0x00007fcffc60741d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
kwho I think you fixed a similar bug elsewhere in a join, so I'll assign to you initially.
Attachments
Issue Links
- blocks
-
IMPALA-6594 Various tests failing (some flaky) with memory limit exceeded
-
- Resolved
-