Details
-
Bug
-
Status: Resolved
-
Critical
-
Resolution: Fixed
-
Impala 3.1.0
-
ghx-label-9
Description
Ran into the following crash on a rhel test recently:
Error starting cluster: num_known_live_backends did not reach expected value in time
Backtrace:
#0 0x00007f92365185c9 in raise () from /lib64/libc.so.6 #1 0x00007f9236519cd8 in abort () from /lib64/libc.so.6 #2 0x00007f92393841a5 in os::abort(bool) () from /opt/toolchain/sun-jdk-64bit-1.8.0.05/jre/lib/amd64/server/libjvm.so #3 0x00007f9239514843 in VMError::report_and_die() () from /opt/toolchain/sun-jdk-64bit-1.8.0.05/jre/lib/amd64/server/libjvm.so #4 0x00007f9239389562 in JVM_handle_linux_signal () from /opt/toolchain/sun-jdk-64bit-1.8.0.05/jre/lib/amd64/server/libjvm.so #5 0x00007f92393804f3 in signalHandler(int, siginfo*, void*) () from /opt/toolchain/sun-jdk-64bit-1.8.0.05/jre/lib/amd64/server/libjvm.so #6 <signal handler called> #7 0x00000000016fded0 in base::subtle::NoBarrier_CompareAndSwap (ptr=0x238, old_value=0, new_value=1) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/gutil/atomicops-internals-x86.h:85 #8 0x00000000016fdf50 in base::subtle::Acquire_CompareAndSwap (ptr=0x238, old_value=0, new_value=1) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/gutil/atomicops-internals-x86.h:138 #9 0x00000000016fe26c in base::SpinLock::Lock (this=0x238) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/gutil/spinlock.h:74 #10 0x00000000016fe2f6 in impala::SpinLock::lock (this=0x238) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/util/spinlock.h:34 #11 0x0000000001aa8c96 in impala::ScopedShardedMapRef<std::shared_ptr<impala::ClientRequestState> >::ScopedShardedMapRef (this=0x7f91aa81eb90, query_id=..., sharded_map=0x1c0) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/util/sharded-query-map-util.h:99 #12 0x0000000001a999e2 in impala::ImpalaServer::GetClientRequestState (this=0xa569000, query_id=...) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/service/impala-server.cc:2123 #13 0x0000000001b3ace6 in impala::ImpalaHttpHandler::QuerySummaryHandler (this=0x6f057a0, include_json_plan=true, include_summary=true, args=..., document=0x7f91aa81f230) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/service/impala-http-handler.cc:755 #14 0x0000000001b3cc11 in impala::ImpalaHttpHandler::<lambda(const auto:5&, auto:6*)>::operator()<std::map<std::basic_string<char>, std::basic_string<char> >, rapidjson::GenericDocument<rapidjson::UTF8<> > >(const std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > *) const (__closure=0xd9884b8, args=..., doc=0x7f91aa81f230) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/service/impala-http-handler.cc:132 #15 0x0000000001b3cc46 in boost::detail::function::void_function_obj_invoker2<impala::ImpalaHttpHandler::RegisterHandlers(impala::Webserver*)::<lambda(const auto:5&, auto:6*)>, void, const std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<const std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >*>::invoke(boost::detail::function::function_buffer &, const std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::le\ ss<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > *) (function_obj_ptr=..., a0=..., a1=0x7f91aa81f230) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153 #16 0x0000000001c4f528 in boost::function2<void, std::map<std::string, std::string, std::less<std::string>, std::allocator<std::pair<std::string const, std::string> > > const&, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >*>::operator() (this=0xd9884b0, a0=..., a1=0x7f91aa81f230) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767 #17 0x0000000001c4d877 in impala::Webserver::RenderUrlWithTemplate (this=0xc590b40, arguments=..., url_handler=..., output=0x7f91aa81f820, content_type=0x7f91aa81f99c) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/util/webserver.cc:440 #18 0x0000000001c4d384 in impala::Webserver::BeginRequestCallback (this=0xc590b40, connection=0xc46c000, request_info=0xc46c000) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/util/webserver.cc:411 #19 0x0000000001c4ce25 in impala::Webserver::BeginRequestCallbackStatic (connection=0xc46c000) at /data/jenkins/workspace/impala-cdh6.x-exhaustive-rhel7/repos/Impala/be/src/util/webserver.cc:370