Details
Major Description
impalad hit a DCHECK failure during the stress test.
F0808 22:23:04.227936 444839 progress-updater.cc:43] Check failed: delta >= 0 (-1 vs. 0)
Stack trace:
#0 0x00000033c7432625 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00000033c7433e05 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x000000000297c854 in google::DumpStackTraceAndExit() ()
incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
#3 0x00000000029732cd in google::LogMessage::Fail() ()
incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
#4 0x0000000002974b72 in google::LogMessage::SendToLog() ()
incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
#5 0x0000000002972ca7 in google::LogMessage::Flush() ()
incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
#6 0x000000000297626e in google::LogMessageFatal::~LogMessageFatal() ()
incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
#7 0x0000000001679a6b in impala::ProgressUpdater::Update (this=0x7f133852b4a8, delta=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/progress-updater.cc:43
_result = 0x7f01d282c080
old_percentage = 6.9028712019688555e-310
num_complete = 139616302725424
new_percentage = 1785311752
#8 0x0000000001b1613d in impala::Coordinator::BackendState::InstanceStats::Update (this=0x7efaf420f8e0, exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:479
children = {<std::_Vector_base<impala::RuntimeProfile*, std::allocator<impala::RuntimeProfile*> >> = {_M_impl = {<std::allocator<impala::RuntimeProfile*>> = {<__gnu_cxx::new_allocator<impala::RuntimeProfile*>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f0e05e3e940, _M_finish = 0x7f0e05e3e968,
_M_end_of_storage = 0x7f0e05e3e980}}, <No data fields>}
thrift_exec_summary = @0x7f133852b4f8
total = 0
delta = -1
#9 0x0000000001b13fcb in impala::Coordinator::BackendState::ApplyExecStatusReport (this=0x7ef1c88a2100, backend_exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8, done=0x7f1216cb403f) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:250
instance_status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0},
static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
instance_idx = 4
instance_stats = 0x7efaf420f8e0
instance_exec_status = @0x7efae12bda00
__for_range = @0x7f1216cb4498
__for_begin = {_M_current = 0x7efae12bda00}
__for_end = {_M_current = 0x7efae12bda80}
l1 = {m = @0x7f133852b5c0}
l2 = {m = @0x7ef1c88a21c0}
#10 0x0000000001aff2db in impala::Coordinator::UpdateBackendExecStatus (this=0x7f133852b000, params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator.cc:951
backend_state = 0x7ef1c88a2100
done = 28
failed_instance_id = {_vptr.TUniqueId = 0x7f1216cb4050, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = "\363\061\065\062\022S\332\353g\260\347\236Al\250\061", hi = 20621415, lo = 27705890}
status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0},
static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f1216cb40af}
#11 0x0000000001528b63 in impala::ImpalaServer::ReportExecStatus (this=0x142f5c00, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-server.cc:1147
request_state = {<std::__shared_ptr<impala::ClientRequestState, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x7f0a0d9dcc00, _M_refcount = {_M_pi = 0x7efc59435800}}, <No data fields>}
#12 0x0000000001599120 in impala::ImpalaInternalService::ReportExecStatus (this=0x14f38e60, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-internal-service.cc:79
No locals.
#13 0x0000000001a6cee8 in impala::ImpalaInternalServiceProcessor::process_ReportExecStatus (this=0x12af0640, seqid=0, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1451
ctx = 0x7f0a0909e7c0
bytes = 0
freer = {handler_ = 0x14f35110, context_ = 0x7f0a0909e7c0, method_ = 0x2f89ec0 "ImpalaInternalService.ReportExecStatus"}
args = {_vptr.ImpalaInternalService_ReportExecStatus_args = 0x3fc0550, params = {_vptr.TReportExecStatusParams = 0x3fac7b0, static ascii_fingerprint = 0x2a19550 "479031394ED7C264DAC814BABAD737AD", static binary_fingerprint = "G\220\061\071N\327\302d\332\310\024\272\272\327\067\255",
protocol_version = impala::ImpalaInternalServiceVersion::V1, query_id = {_vptr.TUniqueId = 0x3fa23b0, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = <same as static member of an already seen type>, hi = 2038521759236003358, lo = -6476124621472006144}, coord_state_idx = 3,
instance_exec_status = {<std::_Vector_base<impala::TFragmentInstanceExecStatus, std::allocator<impala::TFragmentInstanceExecStatus> >> = {_M_impl = {<std::allocator<impala::TFragmentInstanceExecStatus>> = {<__gnu_cxx::new_allocator<impala::TFragmentInstanceExecStatus>> = {<No data fields>}, <No data fields>},
_M_start = 0x7efae12bda00, _M_finish = 0x7efae12bda80, _M_end_of_storage = 0x7efae12bda80}}, <No data fields>}, insert_exec_status = {_vptr.TInsertExecStatus = 0x3fac7d0, static ascii_fingerprint = 0x2a19410 "298B51CA995BEC745FCB22B330B4877B", static binary_fingerprint = <same as static member of an already seen type>,
files_to_move = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<No data fields>}, <No data fields>},
_M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb44c0,
_M_right = 0x7f1216cb44c0}, _M_node_count = 0}}}, per_partition_status = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<No data fields>}, <No data fields>}, _M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0,
_M_left = 0x7f1216cb44f0, _M_right = 0x7f1216cb44f0}, _M_node_count = 0}}}, __isset = {per_partition_status = false}}, error_log = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<No data fields>}, <No data fields>},
_M_key_compare = {<std::binary_function<impala::TErrorCode::type, impala::TErrorCode::type, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb4528, _M_right = 0x7f1216cb4528}, _M_node_count = 0}}}, __isset = {query_id = true, coord_state_idx = true,
instance_exec_status = true, insert_exec_status = false, error_log = false}}, __isset = {params = true}}
result = {_vptr.ImpalaInternalService_ReportExecStatus_result = 0x3fc0510, success = {_vptr.TReportExecStatusResult = 0x3fac790, static ascii_fingerprint = 0x2a191e8 "BA609E39B08A478BB74F0F877E3C5C4A", static binary_fingerprint = "\272`\236\071\260\212G\213\267O\017\207~<\\J", status = {_vptr.TStatus = 0x3fa3850,
static ascii_fingerprint = 0x2a17a80 "A22BE3E84688C9DA4E00CC902B4EE818", static binary_fingerprint = <same as static member of an already seen type>, status_code = impala::TErrorCode::OK,
error_msgs = {<std::_Vector_base<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >> = {
_M_impl = {<std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<__gnu_cxx::new_allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>},
__isset = {error_msgs = false}}, __isset = {status = false}}, __isset = {success = false}}
#14 0x0000000001a6c4ce in impala::ImpalaInternalServiceProcessor::dispatchCall (this=0x12af0640, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, fname=..., seqid=0, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1370
pfn = {_M_node = 0x12af31c0}
#15 0x0000000001200da2 in apache::thrift::TDispatchProcessor::process (this=0x12af0640, in=..., out=..., connectionContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/thrift-0.9.0-p9/include/thrift/TDispatchProcessor.h:121
fname = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f01bde656c8 "ReportExecStatus"}}
mtype = apache::thrift::protocol::T_CALL
seqid = 0
#16 0x00000000013cd0af in apache::thrift::server::TAcceptQueueServer::Task::run (this=0x7eea6ad998a0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/TAcceptQueueServer.cpp:76
eventHandler = {px = 0x11d27980, pn = {pi_ = 0x142bfcc0}}
connectionContext = 0x7eec0fc9dc80
#17 0x00000000013c918b in impala::ThriftThread::RunRunnable (this=0x1637dc40, runnable=..., promise=0x7f1684dff570) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/thrift-thread.cc:64
No locals.
#18 0x00000000013ca8b7 in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0x7f0af77b66f0, p=0x1637dc40, a1=..., a2=0x7f1684dff570)
at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:280
No locals.
#19 0x00000000013ca74d in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0x7f0af77b6700, f=..., a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:392
No locals.
#20 0x00000000013ca499 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0x7f0af77b66f0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
a = {<No data fields>}
#21 0x00000000013ca3ac in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
f = 0x7f0af77b66f0
#22 0x00000000013d7b2c in boost::function0<void>::operator() (this=0x7f1216cb4c20) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
No locals.
#23 0x00000000016ad925 in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f1684dff360) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:330
system_tid = 444839
category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x14d61758 "thrift-server"}}
thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x53b7c00, _M_refcount = {_M_pi = 0x53adc80}}, <No data fields>}
name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f0cd3688858 "backend-5064"}}
#24 0x00000000016b642c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f04da0befc0, f=@0x7f04da0befb8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
No locals.
#25 0x00000000016b636f in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f04da0befb8)
at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
---Type <return> to continue, or q <return> to quit---
a = {<No data fields>}
#26 0x00000000016b6332 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f04da0bee00)
at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
No locals.
#27 0x0000000001ba3dca in thread_proxy ()
No symbol table info available.
#28 0x00000033c7807aa1 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#29 0x00000033c74e893d in clone () from /lib64/libc.so.6
No symbol table info available.