Critical Description
Impalad running stress tests w/ debug libkudu_client.so gets the following DCHECK failure:
F1129 19:32:42.075459 185566 meta_cache.cc:267] Check failed: found Tablet e0eea4076a6843749389956abfed3172: Specified server not found: 520ffc6d07ad42baaf49e59f7129b080 (ve1121.halxg.cloudera.com:7050). Replicas: 275ece6d98a14be9b7dfcee3bec8d7a8 (FOLLOWER, OK), 2d52a82cb62c4064842cf628d01083fe (FOLLOWER, OK), c101883d3e82496989a5f9f667c30e38 (FOLLOWER, OK)
stack from gdb dump:
#0 0x0000003fdd232625 in raise () from /lib64/libc.so.6 #1 0x0000003fdd233e05 in abort () from /lib64/libc.so.6 #2 0x000000000281f3b4 in ?? () #3 0x000000000281881d in google::LogMessage::Fail() () #4 0x000000000281b146 in google::LogMessage::SendToLog() () #5 0x000000000281833d in google::LogMessage::Flush() () #6 0x0000000002818619 in google::LogMessage::~LogMessage() () #7 0x00000000011c902f in impala::LogKuduMessage(void*, kudu::client::KuduLogSeverity, char const*, int, tm const*, char const*, unsigned long) () #8 0x00000000011c9423 in kudu::client::KuduLoggingFunctionCallback<void*>::Run(kudu::client::KuduLogSeverity, char const*, int, tm const*, char const*, unsigned long) () #9 0x00007eff1b24073e in kudu::client::LoggingAdapterCB(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/client.cc:163 #10 0x00007eff1b25863c in kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::client::KuduLoggingCallback* const&, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:584 #11 0x00007eff1b2568fe in kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, kudu::client::KuduLoggingCallback*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:990 #12 0x00007eff1b2540e3 in kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long), void ()(kudu::internal::UnretainedWrapper<kudu::client::KuduLoggingCallback>)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::internal::BindStateBase*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:2114 #13 0x00007eff1b3d1f53 in kudu::Callback<void ()(kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) const () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/callback.h:683 #14 0x00007eff1b3d08bc in kudu::(anonymous namespace)::SimpleSink::send(int, char const*, char const*, int, tm const*, char const*, unsigned long) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/logging.cc:81 #15 0x00007eff1b4ab4a7 in google::LogMessage::SendToLog() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:784 #16 0x00007eff1b4a8df9 in google::LogMessage::Flush() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:1301 #17 0x00007eff1b4ac3ef in google::LogMessageFatal::~LogMessageFatal() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:2013 #18 0x00007eff1b27b3bf in kudu::client::internal::RemoteTablet::MarkTServerAsLeader(kudu::client::internal::RemoteTabletServer const*) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:267 #19 0x00007eff1b27c9f9 in kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void ()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:413 #20 0x00007eff1b237aba in kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/retriable_rpc.h:130 #21 0x00007eff1b2f5531 in kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/rpc.cc:86 #22 0x00007eff1b2f5d68 in boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280 #23 0x00007eff1b2f5c38 in void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:398 #24 0x00007eff1b2f59f2 in void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&&&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1234 #25 0x00007eff1b2f5922 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159 #26 0x00007eff1b2ed176 in boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771 #27 0x00007eff1b2ea18d in kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:490 #28 0x00007eff1b2ee554 in void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &(kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int))>(ev_loop*, ev_timer*, int) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:479 #29 0x00007eff1b51a4cb in ev_invoke_pending () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3155 #30 0x00007eff1b51dbd4 in ev_run () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3555 #31 0x00007eff1b2eb2df in ev::loop_ref::run(int) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:211 #32 0x00007eff1b2e8280 in kudu::rpc::ReactorThread::RunThread() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:317 #33 0x00007eff1b2f3547 in boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49 #34 0x00007eff1b2f3054 in void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:259 #35 0x00007eff1b2f271b in boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1222 #36 0x00007eff1b2f1c20 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159 #37 0x00007eff1b315f3e in boost::function0<void>::operator()() const () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771 #38 0x00007eff1b413434 in kudu::Thread::SuperviseThread(void*) () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/thread.cc:588 #39 0x0000003fdd607aa1 in start_thread () from /lib64/libpthread.so.0 #40 0x0000003fdd2e893d in clone () from /lib64/libc.so.6
Kudu client built at git hash f2aeba
Kudu tserver/master is even more recent (1.2 snapshot), git hash 2f1a2a06d4fe719ef9f35338bc7403915517718a