Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-2364

PHJ DCHECK "state_ == PROCESSING_PROBE (3 vs. 1) "

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Blocker
    • Resolution: Fixed
    • Affects Version/s: Impala 2.2.4
    • Fix Version/s: Impala 2.3.0, Impala 2.2.8
    • Component/s: None
    • Labels:

      Description

      Impalad dcheck during endurance run with "partitioned-hash-join-node-ir.cc:206 Check failed: state_ == PROCESSING_PROBE (3 vs. 1) "
      Stacktrace -

      #0  0x00000031352328e5 in raise () from /lib64/libc.so.6
      #0  0x00000031352328e5 in raise () from /lib64/libc.so.6
      #1  0x00000031352340c5 in abort () from /lib64/libc.so.6
      #2  0x0000000001f3f5f9 in ?? ()
      #3  0x0000000001f36c1d in google::LogMessage::Fail() ()
      #4  0x0000000001f3a7b7 in google::LogMessage::SendToLog() ()
      #5  0x0000000001f391ea in google::LogMessage::Flush() ()
      #6  0x0000000001f394bd in google::LogMessageFatal::~LogMessageFatal() ()
      #7  0x000000000157c8b6 in int impala::PartitionedHashJoinNode::ProcessProbeBatch<0>(impala::RowBatch*, impala::HashTableCtx*)
      #8  0x000000000157bd59 in impala::PartitionedHashJoinNode::ProcessProbeBatch(impala::TJoinOp::type, impala::RowBatch*, impala::HashTableCtx*) ()
      #9  0x000000000156fd72 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) ()
      #10 0x000000000156de81 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) ()
      #11 0x0000000001570103 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) ()
      #12 0x000000000156de81 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) ()
      #13 0x0000000001570103 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) ()
      #14 0x000000000155711c in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) ()
      #15 0x00000000014727a2 in impala::PlanFragmentExecutor::OpenInternal() ()
      #16 0x000000000147264e in impala::PlanFragmentExecutor::Open() ()
      #17 0x00000000010ad6ac in impala::FragmentMgr::FragmentExecState::Exec() ()
      #18 0x00000000010a4af0 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) ()
      #19 0x00000000010a90c4 in boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const ()
      #20 0x00000000010a8e7d in void boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>&, boost::_bi::list0&, int) ()
      #21 0x00000000010a8765 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >::operator()() ()
      #22 0x00000000010a80b3 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >, void>::invoke(boost::detail::function::function_buffer&) ()
      #23 0x0000000000f37e05 in boost::function0<void>::operator()() const ()
      #24 0x0000000001196b78 in impala::Thread::SuperviseThread(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*) ()
      #25 0x000000000119fd14 in void boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0&, int) ()
      #26 0x000000000119fc5b in boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() ()
      #27 0x000000000119fc1a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() ()
      #28 0x0000000001615303 in ?? ()
      #29 0x0000003135607851 in start_thread () from /lib64/libpthread.so.0
      #30 0x00000031352e894d in clone () from /lib64/libc.so.6
      

      core dump is on ve0512.halxg.cloudera.com:/data/1/impala-core-dumps

      Query where this crash happened -

      start query 53 in stream 0 using template query53.tpl
      
      select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1212, 1212 + 1, 1212 + 2, 1212 + 3, 1212 + 4, 1212 + 5, 1212 + 6, 1212 + 7, 1212 + 8, 1212 + 9, 1212 + 10, 1212 + 11) and ((i_category in ('Books', 'Children', 'Electronics') and i_class in ('personal', 'portable', 'reference', 'self-help') and i_brand in ('scholaramalgamalg #14', 'scholaramalgamalg #7', 'exportiunivamalg #9', 'scholaramalgamalg #9')) or (i_category in ('Women', 'Music', 'Men') and i_class in ('accessories', 'classical', 'fragrances', 'pants') and i_brand in ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and ss_sold_date_sk between 2451911 and 2452275 – partition key filter group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales) / avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100; 
      – end query 53 in stream 0 using template query53.tpl
      

      Note that this Impalad wasn't the co-ordinator.

        Attachments

          Activity

            People

            • Assignee:
              ippokratis Ippokratis Pandis
              Reporter:
              ippokratis Ippokratis Pandis
            • Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: