Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.9.0, Impala 2.10.0
-
None
-
ghx-label-2
Description
The ASAN poisoning caught a bug where we return
This was caught when I ran TPCHJoinQueries::test_outer_join. I can reproduce and get a usable backtrace with this:
use tpch_parquet; set mem_limit=1gb; set disable_codegen=1; SELECT straight_join * FROM orders o
RIGHT OUTER JOIN lineitem l ON o.o_orderkey = if(l.l_orderkey % 2 = 0, 0, l.l_orderkey)
ORDER BY l_receiptdate, l_orderkey, l_shipdate
limit 10;
I added some logging to get a backtrace of where the page was unpinned.
E0818 11:25:38.199733 31032 buffer-pool.cc:198] Unpin 0x7fe63c200000@
@ 0x168a077 impala::GetStackTrace()
@ 0x178b6fd impala::BufferPool::Unpin()
@ 0x1d6a0eb impala::BufferedTupleStream::UnpinPageIfNeeded()
@ 0x1d6be36 impala::BufferedTupleStream::NextReadPage()
@ 0x1d756f4 impala::BufferedTupleStream::GetNextInternal<>()
@ 0x1d711b8 impala::BufferedTupleStream::GetNextInternal<>()
@ 0x1d6d721 impala::BufferedTupleStream::GetNext()
@ 0x1a25485 impala::PartitionedHashJoinNode::OutputAllBuild()
@ 0x1a20913 impala::PartitionedHashJoinNode::OutputUnmatchedBuild()
@ 0x1a1dce7 impala::PartitionedHashJoinNode::GetNext()
@ 0x1a6829f impala::TopNNode::Open()
@ 0x147d1f1 impala::FragmentInstanceState::Open()
@ 0x147a9dc impala::FragmentInstanceState::Exec()
@ 0x1428e95 impala::QueryState::ExecFInstance()
@ 0x1369b93 boost::function0<>::operator()()
@ 0x175732e impala::Thread::SuperviseThread()
@ 0x176173b boost::_bi::list4<>::operator()<>()
@ 0x17615c8 boost::_bi::bind_t<>::operator()()
@ 0x1e56e0a thread_proxy
@ 0x7fea0445e6ba start_thread
@ 0x7fea03f7e3dd clone
E0818 11:25:38.240247 30563 buffer-pool.h:443] Poison 0x7fe63c200000
=================================================================
==30176==ERROR: AddressSanitizer: use-after-poison on address 0x7fe63c3cf6f7 at pc 0x000001456d32 bp 0x7fe78ef0e500 sp 0x7fe78ef0e4f8
READ of size 1 at 0x7fe63c3cf6f7 thread T268
#0 0x1456d31 in impala::Tuple::IsNull(impala::NullIndicatorOffset const&) const /tmp/be/src/runtime/tuple.h:195:13
#1 0x1b5f7ec in impala::SlotRef::GetBigIntVal(impala::ScalarExprEvaluator*, impala::TupleRow const*) const /tmp/be/src/exprs/slot-ref.cc:408:22
#2 0x1b585de in impala::ScalarExprEvaluator::GetValue(impala::ScalarExpr const&, impala::TupleRow const*) /tmp/be/src/exprs/scalar-expr-evaluator.cc:298:33
#3 0x145636e in void impala::Tuple::MaterializeExprs<false, true>(impala::TupleRow*, impala::TupleDescriptor const&, impala::ScalarExprEvaluator* const*, impala::MemPool*, impala::StringValue**, int*, int*) /tmp/be/src/runtime/tuple.cc:219:17
#4 0x1a6e2c2 in void impala::Tuple::MaterializeExprs<false, true>(impala::TupleRow*, impala::TupleDescriptor const&, std::vector<impala::ScalarExprEvaluator*, std::allocator<impala::ScalarExprEvaluator*> > const&, impala::MemPool*, std::vector<impala::StringValue*, std::allocator<impala::StringValue*> >*, int*) /tmp/be/src/runtime/tuple.h:155:5
#5 0x1a6d7f0 in impala::TopNNode::InsertTupleRow(impala::TupleRow*) /tmp/be/src/exec/topn-node-ir.cc:40:5
#6 0x1a6d3c5 in impala::TopNNode::InsertBatch(impala::RowBatch*) /tmp/be/src/exec/topn-node-ir.cc:24:5
#7 0x1a68311 in impala::TopNNode::Open(impala::RuntimeState*) /tmp/be/src/exec/topn-node.cc:164:11
#8 0x147d1f0 in impala::FragmentInstanceState::Open() /tmp/be/src/runtime/fragment-instance-state.cc:256:41
#9 0x147a9db in impala::FragmentInstanceState::Exec() /tmp/be/src/runtime/fragment-instance-state.cc:80:12
#10 0x1428e94 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /tmp/be/src/runtime/query-state.cc:366:19
#11 0x1369b92 in boost::function0<void>::operator()() const /tmp/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14
#12 0x175732d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /tmp/be/src/util/thread.cc:329:3
#13 0x176173a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457:9
#14 0x17615c7 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16
#15 0x1e56e09 in thread_proxy (/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x1e56e09)
#16 0x7fea0445e6b9 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76b9)
#17 0x7fea03f7e3dc in clone /build/glibc-bfm8X4/glibc-2.23/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:109
0x7fe63c3cf6f7 is located 1898231 bytes inside of 2097152-byte region [0x7fe63c200000,0x7fe63c400000)
allocated by thread T268 here:
#0 0x1069825 in posix_memalign /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-ubuntu-16-04/toolchain/source/llvm/llvm-3.8.0.src-p1/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:124
#1 0x179f211 in impala::SystemAllocator::AllocateViaMalloc(long, unsigned char**) /tmp/be/src/runtime/bufferpool/system-allocator.cc:122:12
#2 0x179e9c6 in impala::SystemAllocator::Allocate(long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/system-allocator.cc:67:41
#3 0x17a14f3 in impala::BufferPool::BufferAllocator::AllocateInternal(long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-allocator.cc:295:19
#4 0x17a0ac9 in impala::BufferPool::BufferAllocator::Allocate(impala::BufferPool::ClientHandle*, long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-allocator.cc:223:39
#5 0x17899f9 in impala::BufferPool::AllocateBuffer(impala::BufferPool::ClientHandle*, long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-pool.cc:232:19
#6 0x1e04ee4 in impala::Suballocator::AllocateBuffer(long, std::unique_ptr<impala::Suballocation, std::default_delete<impala::Suballocation> >*) /tmp/be/src/runtime/bufferpool/suballocator.cc:98:39
#7 0x1e044e1 in impala::Suballocator::Allocate(long, std::unique_ptr<impala::Suballocation, std::default_delete<impala::Suballocation> >*) /tmp/be/src/runtime/bufferpool/suballocator.cc:64:41
#8 0x1aaabb0 in impala::HashTable::Init(bool*) /tmp/be/src/exec/hash-table.cc:422:39
#9 0x1abda69 in impala::PhjBuilder::Partition::BuildHashTable(bool*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:657:19
#10 0x1abb53b in impala::PhjBuilder::BuildHashTablesAndPrepareProbeStreams() /tmp/be/src/exec/partitioned-hash-join-builder.cc:386:41
#11 0x1aba843 in impala::PhjBuilder::FlushFinal(impala::RuntimeState*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:245:39
#12 0x1abf05f in impala::PhjBuilder::RepartitionBuildInput(impala::PhjBuilder::Partition*, int, impala::BufferedTupleStream*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:551:39
#13 0x1a1beab in impala::PartitionedHashJoinNode::PrepareSpilledPartitionForProbe(impala::RuntimeState*, bool*) /tmp/be/src/exec/partitioned-hash-join-node.cc:429:41
#14 0x1a1ebcd in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) /tmp/be/src/exec/partitioned-hash-join-node.cc:622:41
#15 0x1a6829e in impala::TopNNode::Open(impala::RuntimeState*) /tmp/be/src/exec/topn-node.cc:158:43
#16 0x147d1f0 in impala::FragmentInstanceState::Open() /tmp/be/src/runtime/fragment-instance-state.cc:256:41
#17 0x147a9db in impala::FragmentInstanceState::Exec() /tmp/be/src/runtime/fragment-instance-state.cc:80:12
#18 0x1428e94 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /tmp/be/src/runtime/query-state.cc:366:19
#19 0x1369b92 in boost::function0<void>::operator()() const /tmp/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14
#20 0x175732d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /tmp/be/src/util/thread.cc:329:3
#21 0x176173a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457:9
#22 0x17615c7 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16
#23 0x1e56e09 in thread_proxy (/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x1e56e09)
Attachments
Issue Links
- is broken by
-
-
- Resolved
-