Details
Description
impalad hit a DCHECK failure during the stress test.
F0808 22:23:04.227936 444839 progress-updater.cc:43] Check failed: delta >= 0 (-1 vs. 0)
Stack trace:
#0 0x00000033c7432625 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00000033c7433e05 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x000000000297c854 in google::DumpStackTraceAndExit() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #3 0x00000000029732cd in google::LogMessage::Fail() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #4 0x0000000002974b72 in google::LogMessage::SendToLog() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #5 0x0000000002972ca7 in google::LogMessage::Flush() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #6 0x000000000297626e in google::LogMessageFatal::~LogMessageFatal() () incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484} #7 0x0000000001679a6b in impala::ProgressUpdater::Update (this=0x7f133852b4a8, delta=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/progress-updater.cc:43 _result = 0x7f01d282c080 old_percentage = 6.9028712019688555e-310 num_complete = 139616302725424 new_percentage = 1785311752 #8 0x0000000001b1613d in impala::Coordinator::BackendState::InstanceStats::Update (this=0x7efaf420f8e0, exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:479 children = {<std::_Vector_base<impala::RuntimeProfile*, std::allocator<impala::RuntimeProfile*> >> = {_M_impl = {<std::allocator<impala::RuntimeProfile*>> = {<__gnu_cxx::new_allocator<impala::RuntimeProfile*>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f0e05e3e940, _M_finish = 0x7f0e05e3e968, _M_end_of_storage = 0x7f0e05e3e980}}, <No data fields>} thrift_exec_summary = @0x7f133852b4f8 total = 0 delta = -1 #9 0x0000000001b13fcb in impala::Coordinator::BackendState::ApplyExecStatusReport (this=0x7ef1c88a2100, backend_exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8, done=0x7f1216cb403f) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:250 instance_status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0} instance_idx = 4 instance_stats = 0x7efaf420f8e0 instance_exec_status = @0x7efae12bda00 __for_range = @0x7f1216cb4498 __for_begin = {_M_current = 0x7efae12bda00} __for_end = {_M_current = 0x7efae12bda80} l1 = {m = @0x7f133852b5c0} l2 = {m = @0x7ef1c88a21c0} #10 0x0000000001aff2db in impala::Coordinator::UpdateBackendExecStatus (this=0x7f133852b000, params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator.cc:951 backend_state = 0x7ef1c88a2100 done = 28 failed_instance_id = {_vptr.TUniqueId = 0x7f1216cb4050, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = "\363\061\065\062\022S\332\353g\260\347\236Al\250\061", hi = 20621415, lo = 27705890} status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0}, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f1216cb40af} #11 0x0000000001528b63 in impala::ImpalaServer::ReportExecStatus (this=0x142f5c00, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-server.cc:1147 request_state = {<std::__shared_ptr<impala::ClientRequestState, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x7f0a0d9dcc00, _M_refcount = {_M_pi = 0x7efc59435800}}, <No data fields>} #12 0x0000000001599120 in impala::ImpalaInternalService::ReportExecStatus (this=0x14f38e60, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-internal-service.cc:79 No locals. #13 0x0000000001a6cee8 in impala::ImpalaInternalServiceProcessor::process_ReportExecStatus (this=0x12af0640, seqid=0, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1451 ctx = 0x7f0a0909e7c0 bytes = 0 freer = {handler_ = 0x14f35110, context_ = 0x7f0a0909e7c0, method_ = 0x2f89ec0 "ImpalaInternalService.ReportExecStatus"} args = {_vptr.ImpalaInternalService_ReportExecStatus_args = 0x3fc0550, params = {_vptr.TReportExecStatusParams = 0x3fac7b0, static ascii_fingerprint = 0x2a19550 "479031394ED7C264DAC814BABAD737AD", static binary_fingerprint = "G\220\061\071N\327\302d\332\310\024\272\272\327\067\255", protocol_version = impala::ImpalaInternalServiceVersion::V1, query_id = {_vptr.TUniqueId = 0x3fa23b0, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = <same as static member of an already seen type>, hi = 2038521759236003358, lo = -6476124621472006144}, coord_state_idx = 3, instance_exec_status = {<std::_Vector_base<impala::TFragmentInstanceExecStatus, std::allocator<impala::TFragmentInstanceExecStatus> >> = {_M_impl = {<std::allocator<impala::TFragmentInstanceExecStatus>> = {<__gnu_cxx::new_allocator<impala::TFragmentInstanceExecStatus>> = {<No data fields>}, <No data fields>}, _M_start = 0x7efae12bda00, _M_finish = 0x7efae12bda80, _M_end_of_storage = 0x7efae12bda80}}, <No data fields>}, insert_exec_status = {_vptr.TInsertExecStatus = 0x3fac7d0, static ascii_fingerprint = 0x2a19410 "298B51CA995BEC745FCB22B330B4877B", static binary_fingerprint = <same as static member of an already seen type>, files_to_move = {_M_t = { _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<No data fields>}, <No data fields>}, _M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb44c0, _M_right = 0x7f1216cb44c0}, _M_node_count = 0}}}, per_partition_status = {_M_t = { _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<No data fields>}, <No data fields>}, _M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb44f0, _M_right = 0x7f1216cb44f0}, _M_node_count = 0}}}, __isset = {per_partition_status = false}}, error_log = {_M_t = { _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<No data fields>}, <No data fields>}, _M_key_compare = {<std::binary_function<impala::TErrorCode::type, impala::TErrorCode::type, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb4528, _M_right = 0x7f1216cb4528}, _M_node_count = 0}}}, __isset = {query_id = true, coord_state_idx = true, instance_exec_status = true, insert_exec_status = false, error_log = false}}, __isset = {params = true}} result = {_vptr.ImpalaInternalService_ReportExecStatus_result = 0x3fc0510, success = {_vptr.TReportExecStatusResult = 0x3fac790, static ascii_fingerprint = 0x2a191e8 "BA609E39B08A478BB74F0F877E3C5C4A", static binary_fingerprint = "\272`\236\071\260\212G\213\267O\017\207~<\\J", status = {_vptr.TStatus = 0x3fa3850, static ascii_fingerprint = 0x2a17a80 "A22BE3E84688C9DA4E00CC902B4EE818", static binary_fingerprint = <same as static member of an already seen type>, status_code = impala::TErrorCode::OK, error_msgs = {<std::_Vector_base<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >> = { _M_impl = {<std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<__gnu_cxx::new_allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>}, __isset = {error_msgs = false}}, __isset = {status = false}}, __isset = {success = false}} #14 0x0000000001a6c4ce in impala::ImpalaInternalServiceProcessor::dispatchCall (this=0x12af0640, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, fname=..., seqid=0, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1370 pfn = {_M_node = 0x12af31c0} #15 0x0000000001200da2 in apache::thrift::TDispatchProcessor::process (this=0x12af0640, in=..., out=..., connectionContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/thrift-0.9.0-p9/include/thrift/TDispatchProcessor.h:121 fname = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f01bde656c8 "ReportExecStatus"}} mtype = apache::thrift::protocol::T_CALL seqid = 0 #16 0x00000000013cd0af in apache::thrift::server::TAcceptQueueServer::Task::run (this=0x7eea6ad998a0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/TAcceptQueueServer.cpp:76 eventHandler = {px = 0x11d27980, pn = {pi_ = 0x142bfcc0}} connectionContext = 0x7eec0fc9dc80 #17 0x00000000013c918b in impala::ThriftThread::RunRunnable (this=0x1637dc40, runnable=..., promise=0x7f1684dff570) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/thrift-thread.cc:64 No locals. #18 0x00000000013ca8b7 in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0x7f0af77b66f0, p=0x1637dc40, a1=..., a2=0x7f1684dff570) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:280 No locals. #19 0x00000000013ca74d in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0x7f0af77b6700, f=..., a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:392 No locals. #20 0x00000000013ca499 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0x7f0af77b66f0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20 a = {<No data fields>} #21 0x00000000013ca3ac in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153 f = 0x7f0af77b66f0 #22 0x00000000013d7b2c in boost::function0<void>::operator() (this=0x7f1216cb4c20) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767 No locals. #23 0x00000000016ad925 in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f1684dff360) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:330 system_tid = 444839 category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x14d61758 "thrift-server"}} thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x53b7c00, _M_refcount = {_M_pi = 0x53adc80}}, <No data fields>} name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f0cd3688858 "backend-5064"}} #24 0x00000000016b642c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f04da0befc0, f=@0x7f04da0befb8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457 No locals. #25 0x00000000016b636f in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f04da0befb8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20 ---Type <return> to continue, or q <return> to quit--- a = {<No data fields>} #26 0x00000000016b6332 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f04da0bee00) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116 No locals. #27 0x0000000001ba3dca in thread_proxy () No symbol table info available. #28 0x00000033c7807aa1 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #29 0x00000033c74e893d in clone () from /lib64/libc.so.6 No symbol table info available.