Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.2.4
-
None
Description
Impalad dcheck during endurance run with "partitioned-hash-join-node-ir.cc:206 Check failed: state_ == PROCESSING_PROBE (3 vs. 1) "
Stacktrace -
#0 0x00000031352328e5 in raise () from /lib64/libc.so.6 #0 0x00000031352328e5 in raise () from /lib64/libc.so.6 #1 0x00000031352340c5 in abort () from /lib64/libc.so.6 #2 0x0000000001f3f5f9 in ?? () #3 0x0000000001f36c1d in google::LogMessage::Fail() () #4 0x0000000001f3a7b7 in google::LogMessage::SendToLog() () #5 0x0000000001f391ea in google::LogMessage::Flush() () #6 0x0000000001f394bd in google::LogMessageFatal::~LogMessageFatal() () #7 0x000000000157c8b6 in int impala::PartitionedHashJoinNode::ProcessProbeBatch<0>(impala::RowBatch*, impala::HashTableCtx*) #8 0x000000000157bd59 in impala::PartitionedHashJoinNode::ProcessProbeBatch(impala::TJoinOp::type, impala::RowBatch*, impala::HashTableCtx*) () #9 0x000000000156fd72 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #10 0x000000000156de81 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #11 0x0000000001570103 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #12 0x000000000156de81 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #13 0x0000000001570103 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #14 0x000000000155711c in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #15 0x00000000014727a2 in impala::PlanFragmentExecutor::OpenInternal() () #16 0x000000000147264e in impala::PlanFragmentExecutor::Open() () #17 0x00000000010ad6ac in impala::FragmentMgr::FragmentExecState::Exec() () #18 0x00000000010a4af0 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #19 0x00000000010a90c4 in boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #20 0x00000000010a8e7d in void boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>&, boost::_bi::list0&, int) () #21 0x00000000010a8765 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >::operator()() () #22 0x00000000010a80b3 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >, void>::invoke(boost::detail::function::function_buffer&) () #23 0x0000000000f37e05 in boost::function0<void>::operator()() const () #24 0x0000000001196b78 in impala::Thread::SuperviseThread(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*) () #25 0x000000000119fd14 in void boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0&, int) () #26 0x000000000119fc5b in boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() () #27 0x000000000119fc1a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() () #28 0x0000000001615303 in ?? () #29 0x0000003135607851 in start_thread () from /lib64/libpthread.so.0 #30 0x00000031352e894d in clone () from /lib64/libc.so.6
core dump is on ve0512.halxg.cloudera.com:/data/1/impala-core-dumps
Query where this crash happened -
start query 53 in stream 0 using template query53.tpl
select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1212, 1212 + 1, 1212 + 2, 1212 + 3, 1212 + 4, 1212 + 5, 1212 + 6, 1212 + 7, 1212 + 8, 1212 + 9, 1212 + 10, 1212 + 11) and ((i_category in ('Books', 'Children', 'Electronics') and i_class in ('personal', 'portable', 'reference', 'self-help') and i_brand in ('scholaramalgamalg #14', 'scholaramalgamalg #7', 'exportiunivamalg #9', 'scholaramalgamalg #9')) or (i_category in ('Women', 'Music', 'Men') and i_class in ('accessories', 'classical', 'fragrances', 'pants') and i_brand in ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and ss_sold_date_sk between 2451911 and 2452275 – partition key filter group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales) / avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100;
– end query 53 in stream 0 using template query53.tpl
Note that this Impalad wasn't the co-ordinator.