Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-4748

tmp-file-mgr.h:263] Check failed: !write_in_flight_

    XMLWordPrintableJSON

Details

    Description

      While running a stress test, Impala hit a DCHECK:

      259   class WriteHandle {
260    public:
261     // The write must be destroyed by FileGroup::DestroyWriteHandle().
262     ~WriteHandle() {
263       DCHECK(!write_in_flight_);
264       DCHECK(is_cancelled_);
265     }

      #6  0x000000000282d98e in google::LogMessageFatal::~LogMessageFatal() ()
#7  0x000000000138bbd7 in impala::TmpFileMgr::WriteHandle::~WriteHandle (this=0x7f1b46dae5a0, __in_chrg=<value optimized out>)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/tmp-file-mgr.h:263
#8  0x000000000138ef0a in std::default_delete<impala::TmpFileMgr::WriteHandle>::operator() (this=0x7f244fd296b0,
    __ptr=0x7f1b46dae5a0)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:76
#9  0x000000000138d209 in std::unique_ptr<impala::TmpFileMgr::WriteHandle, std::default_delete<impala::TmpFileMgr::WriteHandle> >::~unique_ptr (this=0x7f244fd296b0, __in_chrg=<value optimized out>)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:236
#10 0x00000000013887de in impala::TmpFileMgr::FileGroup::Write (this=0x7f187d5472a0, buffer=..., cb=..., handle=0x7f155340e718)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/tmp-file-mgr.cc:377
#11 0x00000000013992b2 in impala::BufferedBlockMgr::WriteUnpinnedBlock (this=0x7f1b1906ea80, block=0x7f155340e6e0)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-block-mgr.cc:752
#12 0x0000000001398bd4 in impala::BufferedBlockMgr::WriteUnpinnedBlocks (this=0x7f1b1906ea80)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-block-mgr.cc:737
#13 0x000000000139cbbb in impala::BufferedBlockMgr::FindBufferForBlock (this=0x7f1b1906ea80, block=0x7f113936c5d0,
    in_mem=0x7f244fd29f0f) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-block-mgr.cc:1026
#14 0x00000000013954df in impala::BufferedBlockMgr::GetNewBlock (this=0x7f1b1906ea80, client=0x7f18225a7980, unpin_block=0x0,
    block=0x7f244fd2a1d8, len=8388608) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-block-mgr.cc:442
#15 0x000000000199b904 in impala::BufferedTupleStream::NewWriteBlock (this=0x7f1d84310940, block_len=8388608,
    null_indicators_size=0, got_block=0x7f244fd2a47f)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-tuple-stream.cc:246
#16 0x000000000199af09 in impala::BufferedTupleStream::SwitchToIoBuffers (this=0x7f1d84310940, got_buffer=0x7f244fd2a47f)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/buffered-tuple-stream.cc:185
#17 0x00000000017ee8e3 in impala::PhjBuilder::AppendRowStreamFull (this=0x7f14a820e540, stream=0x7f1d84310940,
    row=0x7f1698a7acd0, status=0x7f244fd2a660)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-builder.cc:277
#18 0x00007f25408e0fa3 in impala::PhjBuilder::ProcessBuildBatch ()
#19 0x00000000017ed7de in impala::PhjBuilder::Send (this=0x7f14a820e540, state=0x7f15977e9c00, batch=0xd6e91a80)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-builder.cc:174
#20 0x00000000017cdf9f in impala::BlockingJoinNode::SendBuildInputToSink<false> (this=0x7f14999be300, state=0x7f15977e9c00,
    build_sink=0x7f14a820e540) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/blocking-join-node.cc:287
#21 0x00000000017cc08a in impala::BlockingJoinNode::ProcessBuildInputAndOpenProbe (this=0x7f14999be300, state=0x7f15977e9c00,
    build_sink=0x7f14a820e540) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/blocking-join-node.cc:233
#22 0x000000000176db9a in impala::PartitionedHashJoinNode::Open (this=0x7f14999be300, state=0x7f15977e9c00)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-node.cc:182
#23 0x00000000017cc136 in impala::BlockingJoinNode::ProcessBuildInputAndOpenProbe (this=0x7f12e7926a00, state=0x7f15977e9c00,
    build_sink=0x7f12d20e3640) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/blocking-join-node.cc:235
#24 0x000000000176db9a in impala::PartitionedHashJoinNode::Open (this=0x7f12e7926a00, state=0x7f15977e9c00)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-node.cc:182
#25 0x00000000017cc136 in impala::BlockingJoinNode::ProcessBuildInputAndOpenProbe (this=0x7f12e7925880, state=0x7f15977e9c00,
    build_sink=0x7f18a6442c40) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/blocking-join-node.cc:235
#26 0x000000000176db9a in impala::PartitionedHashJoinNode::Open (this=0x7f12e7925880, state=0x7f15977e9c00)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-node.cc:182
#27 0x00000000017cc136 in impala::BlockingJoinNode::ProcessBuildInputAndOpenProbe (this=0x7f202baf0000, state=0x7f15977e9c00,
    build_sink=0x10f605480) at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/blocking-join-node.cc:235
#28 0x000000000176db9a in impala::PartitionedHashJoinNode::Open (this=0x7f202baf0000, state=0x7f15977e9c00)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-hash-join-node.cc:182
#29 0x000000000174cc12 in impala::PartitionedAggregationNode::Open (this=0x7f1f2383f840, state=0x7f15977e9c00)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/exec/partitioned-aggregation-node.cc:318
#30 0x00000000019eda6c in impala::PlanFragmentExecutor::OpenInternal (this=0x7f1e6c8cbab0)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.cc:325
#31 0x00000000019ed4a8 in impala::PlanFragmentExecutor::Open (this=0x7f1e6c8cbab0)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.cc:297
#32 0x00000000019e755f in impala::FragmentInstanceState::Exec (this=0x7f1e6c8cb800)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/fragment-instance-state.cc:65
#33 0x00000000019f2c4d in impala::QueryExecMgr::ExecFInstance (this=0xe268340, fis=0x7f1e6c8cb800)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/runtime/query-exec-mgr.cc:100
#34 0x00000000019f6198 in boost::_mfi::mf1<void, impala::QueryExecMgr, impala::FragmentInstanceState*>::operator() (
    this=0x7f12d0ff99e0, p=0xe268340, a1=0x7f1e6c8cb800)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:165
#35 0x00000000019f603b in boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>, boost::_bi::value<impala::FragmentInstanceState*> >::operator()<boost::_mfi::mf1<void, impala::QueryExecMgr, impala::FragmentInstanceState*>, boost::_bi::list0> (
    this=0x7f12d0ff99f0, f=..., a=...)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:313
#36 0x00000000019f5cff in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::QueryExecMgr, impala::FragmentInstanceState*>, boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>, boost::_bi::value<impala::FragmentInstanceState*> > >::operator() (
    this=0x7f12d0ff99e0)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
#37 0x00000000019f57d6 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::QueryExecMgr, impala::FragmentInstanceState*>, boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>, boost::_bi::value<impala::FragmentInstanceState*> > >, void>::invoke (function_obj_ptr=...)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:153
#38 0x000000000134c614 in boost::function0<void>::operator() (this=0x7f244fd2cc40)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:767
#39 0x00000000015f580f in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f1a72857870)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/be/src/util/thread.cc:317
#40 0x00000000015fc7e8 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f12d34b63c0,
    f=@0x7f12d34b63b8, a=...)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:457
#41 0x00000000015fc72b in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f12d34b63b8)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
#42 0x00000000015fc686 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function <void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f12d34b6200)
    at /usr/src/debug/impala-2.7.0-cdh5.11.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
#43 0x0000000001a5bc4a in thread_proxy ()
#44 0x00000035202079d1 in start_thread () from /lib64/libpthread.so.0
#45 0x000000351fee88fd in clone () from /lib64/libc.so.6

      Commit: IMPALA-3401: Suppress blocks of content devoted to Cloudera Manager. (4 days ago) <John Russell>

      Attachments

        Activity

          People

            tarmstrong Tim Armstrong
            mikeb Michael Brown
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: