Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-5815

Right outer join returns reference to unpinned memory

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Blocker
    • Resolution: Fixed
    • Affects Version/s: Impala 2.9.0, Impala 2.10.0
    • Fix Version/s: Impala 2.10.0
    • Component/s: Backend
    • Labels:
      None

      Description

      The ASAN poisoning caught a bug where we return

      This was caught when I ran TPCHJoinQueries::test_outer_join. I can reproduce and get a usable backtrace with this:

      use tpch_parquet; set mem_limit=1gb; set disable_codegen=1; SELECT straight_join * FROM orders o
      RIGHT OUTER JOIN lineitem l ON o.o_orderkey =  if(l.l_orderkey % 2 = 0, 0, l.l_orderkey)
      ORDER BY l_receiptdate, l_orderkey, l_shipdate
      limit 10;
      

      I added some logging to get a backtrace of where the page was unpinned.

      E0818 11:25:38.199733 31032 buffer-pool.cc:198] Unpin 0x7fe63c200000@ 
          @          0x168a077  impala::GetStackTrace()
          @          0x178b6fd  impala::BufferPool::Unpin()
          @          0x1d6a0eb  impala::BufferedTupleStream::UnpinPageIfNeeded()
          @          0x1d6be36  impala::BufferedTupleStream::NextReadPage()
          @          0x1d756f4  impala::BufferedTupleStream::GetNextInternal<>()
          @          0x1d711b8  impala::BufferedTupleStream::GetNextInternal<>()
          @          0x1d6d721  impala::BufferedTupleStream::GetNext()
          @          0x1a25485  impala::PartitionedHashJoinNode::OutputAllBuild()
          @          0x1a20913  impala::PartitionedHashJoinNode::OutputUnmatchedBuild()
          @          0x1a1dce7  impala::PartitionedHashJoinNode::GetNext()
          @          0x1a6829f  impala::TopNNode::Open()
          @          0x147d1f1  impala::FragmentInstanceState::Open()
          @          0x147a9dc  impala::FragmentInstanceState::Exec()
          @          0x1428e95  impala::QueryState::ExecFInstance()
          @          0x1369b93  boost::function0<>::operator()()
          @          0x175732e  impala::Thread::SuperviseThread()
          @          0x176173b  boost::_bi::list4<>::operator()<>()
          @          0x17615c8  boost::_bi::bind_t<>::operator()()
          @          0x1e56e0a  thread_proxy
          @     0x7fea0445e6ba  start_thread
          @     0x7fea03f7e3dd  clone
      E0818 11:25:38.240247 30563 buffer-pool.h:443] Poison 0x7fe63c200000
      =================================================================
      ==30176==ERROR: AddressSanitizer: use-after-poison on address 0x7fe63c3cf6f7 at pc 0x000001456d32 bp 0x7fe78ef0e500 sp 0x7fe78ef0e4f8
      READ of size 1 at 0x7fe63c3cf6f7 thread T268
          #0 0x1456d31 in impala::Tuple::IsNull(impala::NullIndicatorOffset const&) const /tmp/be/src/runtime/tuple.h:195:13
          #1 0x1b5f7ec in impala::SlotRef::GetBigIntVal(impala::ScalarExprEvaluator*, impala::TupleRow const*) const /tmp/be/src/exprs/slot-ref.cc:408:22
          #2 0x1b585de in impala::ScalarExprEvaluator::GetValue(impala::ScalarExpr const&, impala::TupleRow const*) /tmp/be/src/exprs/scalar-expr-evaluator.cc:298:33
          #3 0x145636e in void impala::Tuple::MaterializeExprs<false, true>(impala::TupleRow*, impala::TupleDescriptor const&, impala::ScalarExprEvaluator* const*, impala::MemPool*, impala::StringValue**, int*, int*) /tmp/be/src/runtime/tuple.cc:219:17
          #4 0x1a6e2c2 in void impala::Tuple::MaterializeExprs<false, true>(impala::TupleRow*, impala::TupleDescriptor const&, std::vector<impala::ScalarExprEvaluator*, std::allocator<impala::ScalarExprEvaluator*> > const&, impala::MemPool*, std::vector<impala::StringValue*, std::allocator<impala::StringValue*> >*, int*) /tmp/be/src/runtime/tuple.h:155:5
          #5 0x1a6d7f0 in impala::TopNNode::InsertTupleRow(impala::TupleRow*) /tmp/be/src/exec/topn-node-ir.cc:40:5
          #6 0x1a6d3c5 in impala::TopNNode::InsertBatch(impala::RowBatch*) /tmp/be/src/exec/topn-node-ir.cc:24:5
          #7 0x1a68311 in impala::TopNNode::Open(impala::RuntimeState*) /tmp/be/src/exec/topn-node.cc:164:11
          #8 0x147d1f0 in impala::FragmentInstanceState::Open() /tmp/be/src/runtime/fragment-instance-state.cc:256:41
          #9 0x147a9db in impala::FragmentInstanceState::Exec() /tmp/be/src/runtime/fragment-instance-state.cc:80:12
          #10 0x1428e94 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /tmp/be/src/runtime/query-state.cc:366:19
          #11 0x1369b92 in boost::function0<void>::operator()() const /tmp/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14
          #12 0x175732d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /tmp/be/src/util/thread.cc:329:3
          #13 0x176173a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457:9
          #14 0x17615c7 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16
          #15 0x1e56e09 in thread_proxy (/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x1e56e09)
          #16 0x7fea0445e6b9 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76b9)
          #17 0x7fea03f7e3dc in clone /build/glibc-bfm8X4/glibc-2.23/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:109
      
      0x7fe63c3cf6f7 is located 1898231 bytes inside of 2097152-byte region [0x7fe63c200000,0x7fe63c400000)
      allocated by thread T268 here:
          #0 0x1069825 in posix_memalign /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-ubuntu-16-04/toolchain/source/llvm/llvm-3.8.0.src-p1/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:124
          #1 0x179f211 in impala::SystemAllocator::AllocateViaMalloc(long, unsigned char**) /tmp/be/src/runtime/bufferpool/system-allocator.cc:122:12
          #2 0x179e9c6 in impala::SystemAllocator::Allocate(long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/system-allocator.cc:67:41
          #3 0x17a14f3 in impala::BufferPool::BufferAllocator::AllocateInternal(long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-allocator.cc:295:19
          #4 0x17a0ac9 in impala::BufferPool::BufferAllocator::Allocate(impala::BufferPool::ClientHandle*, long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-allocator.cc:223:39
          #5 0x17899f9 in impala::BufferPool::AllocateBuffer(impala::BufferPool::ClientHandle*, long, impala::BufferPool::BufferHandle*) /tmp/be/src/runtime/bufferpool/buffer-pool.cc:232:19
          #6 0x1e04ee4 in impala::Suballocator::AllocateBuffer(long, std::unique_ptr<impala::Suballocation, std::default_delete<impala::Suballocation> >*) /tmp/be/src/runtime/bufferpool/suballocator.cc:98:39
          #7 0x1e044e1 in impala::Suballocator::Allocate(long, std::unique_ptr<impala::Suballocation, std::default_delete<impala::Suballocation> >*) /tmp/be/src/runtime/bufferpool/suballocator.cc:64:41
          #8 0x1aaabb0 in impala::HashTable::Init(bool*) /tmp/be/src/exec/hash-table.cc:422:39
          #9 0x1abda69 in impala::PhjBuilder::Partition::BuildHashTable(bool*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:657:19
          #10 0x1abb53b in impala::PhjBuilder::BuildHashTablesAndPrepareProbeStreams() /tmp/be/src/exec/partitioned-hash-join-builder.cc:386:41
          #11 0x1aba843 in impala::PhjBuilder::FlushFinal(impala::RuntimeState*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:245:39
          #12 0x1abf05f in impala::PhjBuilder::RepartitionBuildInput(impala::PhjBuilder::Partition*, int, impala::BufferedTupleStream*) /tmp/be/src/exec/partitioned-hash-join-builder.cc:551:39
          #13 0x1a1beab in impala::PartitionedHashJoinNode::PrepareSpilledPartitionForProbe(impala::RuntimeState*, bool*) /tmp/be/src/exec/partitioned-hash-join-node.cc:429:41
          #14 0x1a1ebcd in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) /tmp/be/src/exec/partitioned-hash-join-node.cc:622:41
          #15 0x1a6829e in impala::TopNNode::Open(impala::RuntimeState*) /tmp/be/src/exec/topn-node.cc:158:43
          #16 0x147d1f0 in impala::FragmentInstanceState::Open() /tmp/be/src/runtime/fragment-instance-state.cc:256:41
          #17 0x147a9db in impala::FragmentInstanceState::Exec() /tmp/be/src/runtime/fragment-instance-state.cc:80:12
          #18 0x1428e94 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /tmp/be/src/runtime/query-state.cc:366:19
          #19 0x1369b92 in boost::function0<void>::operator()() const /tmp/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14
          #20 0x175732d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /tmp/be/src/util/thread.cc:329:3
          #21 0x176173a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457:9
          #22 0x17615c7 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /tmp/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16
          #23 0x1e56e09 in thread_proxy (/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x1e56e09)
      

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                tarmstrong Tim Armstrong
                Reporter:
                tarmstrong Tim Armstrong
              • Votes:
                0 Vote for this issue
                Watchers:
                2 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: