1852 threads backend #0 0x00000035e240eb5c in recv () from /lib64/libpthread.so.0 #1 0x0000000001f9819d in apache::thrift::transport::TSocket::read(unsigned char*, unsigned int) () #2 0x0000000001fa16d6 in apache::thrift::transport::TBufferedTransport::peek() () #3 0x0000000001fa7a5f in apache::thrift::server::TThreadedServer::Task::run() () #4 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #5 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #6 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #7 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #8 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #9 0x00000000011ca619 in boost::function0::operator()() const () #10 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #11 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #12 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #13 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #14 0x00000000016d8c53 in ?? () #15 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #16 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 418 threads SenderThread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000169fd4e in impala::BlockingQueue::BlockingGet(impala::TRowBatch**) () #3 0x000000000169ef5d in impala::ThreadPool::WorkerThread(int) () #4 0x00000000016a1879 in boost::_mfi::mf1, int>::operator()(impala::ThreadPool*, int) const () #5 0x00000000016a16f1 in void boost::_bi::list2*>, boost::_bi::value >::operator(), int>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1, int>&, boost::_bi::list0&, int) () #6 0x00000000016a1417 in boost::_bi::bind_t, int>, boost::_bi::list2*>, boost::_bi::value > >::operator()() () #7 0x00000000016a0f68 in boost::detail::function::void_function_obj_invoker0, int>, boost::_bi::list2*>, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 135 threads backend #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000151a696 in impala::DataStreamRecvr::SenderQueue::AddBatch(impala::TRowBatch const&) () #3 0x000000000151c15c in impala::DataStreamRecvr::AddBatch(impala::TRowBatch const&, int) () #4 0x0000000001515a95 in impala::DataStreamMgr::AddData(impala::TUniqueId const&, int, impala::TRowBatch const&, int) () #5 0x000000000125b487 in impala::ImpalaServer::TransmitData(impala::TTransmitDataResult&, impala::TTransmitDataParams const&) () #6 0x000000000126ea55 in impala::ImpalaInternalService::TransmitData(impala::TTransmitDataResult&, impala::TTransmitDataParams const&) () #7 0x000000000144c973 in impala::ImpalaInternalServiceProcessor::process_TransmitData(int, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, void*) () #8 0x000000000144ae94 in impala::ImpalaInternalServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::basic_string, std::allocator > const&, int, void*) () #9 0x00000000012647d2 in apache::thrift::TDispatchProcessor::process(boost::shared_ptr, boost::shared_ptr, void*) () #10 0x0000000001fa7a19 in apache::thrift::server::TThreadedServer::Task::run() () #11 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #12 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #13 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #14 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #15 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #16 0x00000000011ca619 in boost::function0::operator()() const () #17 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #18 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #19 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #20 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #21 0x00000000016d8c53 in ?? () #22 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #23 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 66 threads report-profile #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000118a6fe in boost::condition_variable::do_wait_until(boost::unique_lock&, timespec const&) () #2 0x000000000118a54b in boost::condition_variable::timed_wait(boost::unique_lock&, boost::posix_time::ptime const&) () #3 0x0000000001526e8e in impala::PlanFragmentExecutor::ReportProfile() () #4 0x000000000152c591 in boost::_mfi::mf0::operator()(impala::PlanFragmentExecutor*) const () #5 0x000000000152c266 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x000000000152bda3 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x000000000152ba5f in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 64 threads beeswax-frontend #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000001fa98f7 in apache::thrift::concurrency::Monitor::wait(long) const () #2 0x0000000001f920c4 in apache::thrift::concurrency::ThreadManager::Worker::run() () #3 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #4 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #5 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #6 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #7 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 64 threads hiveserver2-frontend #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000001fa98f7 in apache::thrift::concurrency::Monitor::wait(long) const () #2 0x0000000001f920c4 in apache::thrift::concurrency::ThreadManager::Worker::run() () #3 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #4 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #5 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #6 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #7 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 36 threads work-loop #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x00000000012168fb in impala::DiskIoMgr::GetNextRequestRange(impala::DiskIoMgr::DiskQueue*, impala::DiskIoMgr::RequestRange**, impala::DiskIoMgr::RequestContext**) () #3 0x0000000001217d98 in impala::DiskIoMgr::WorkLoop(impala::DiskIoMgr::DiskQueue*) () #4 0x0000000001225b6e in boost::_mfi::mf1::operator()(impala::DiskIoMgr*, impala::DiskIoMgr::DiskQueue*) const () #5 0x000000000122574b in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #6 0x0000000001224d8b in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #7 0x000000000122438e in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 31 threads SenderThread #0 0x00000035e240eb5c in recv () from /lib64/libpthread.so.0 #1 0x0000000001f9819d in apache::thrift::transport::TSocket::read(unsigned char*, unsigned int) () #2 0x0000000001fa0ae1 in apache::thrift::transport::TBufferedTransport::readSlow(unsigned char*, unsigned int) () #3 0x0000000000f34554 in apache::thrift::transport::TBufferBase::read(unsigned char*, unsigned int) () #4 0x0000000000f36ef0 in unsigned int apache::thrift::transport::readAll(apache::thrift::transport::TBufferBase&, unsigned char*, unsigned int) () #5 0x0000000000f345da in apache::thrift::transport::TBufferBase::readAll(unsigned char*, unsigned int) () #6 0x000000000115f25f in apache::thrift::transport::TBufferedTransport::readAll(unsigned char*, unsigned int) () #7 0x0000000001168c79 in apache::thrift::transport::TVirtualTransport::readAll_virt(unsigned char*, unsigned int) () #8 0x000000000117bd2f in apache::thrift::transport::TTransport::readAll(unsigned char*, unsigned int) () #9 0x00000000011844b8 in apache::thrift::protocol::TBinaryProtocolT::readI32(int&) () #10 0x0000000001183bb9 in apache::thrift::protocol::TBinaryProtocolT::readMessageBegin(std::basic_string, std::allocator >&, apache::thrift::protocol::TMessageType&, int&) () #11 0x0000000001180190 in apache::thrift::protocol::TVirtualProtocol, apache::thrift::protocol::TProtocolDefaults>::readMessageBegin_virt(std::basic_string, std::allocator >&, apache::thrift::protocol::TMessageType&, int&) () #12 0x00000000012645d4 in apache::thrift::protocol::TProtocol::readMessageBegin(std::basic_string, std::allocator >&, apache::thrift::protocol::TMessageType&, int&) () #13 0x000000000144a664 in impala::ImpalaInternalServiceClient::recv_TransmitData(impala::TTransmitDataResult&) () #14 0x000000000144a436 in impala::ImpalaInternalServiceClient::TransmitData(impala::TTransmitDataResult&, impala::TTransmitDataParams const&) () #15 0x000000000169e6ac in impala::Status impala::ClientConnection::DoRpc(void (impala::ImpalaInternalServiceClient::* const&)(impala::TTransmitDataResult&, impala::TTransmitDataParams const&), impala::TTransmitDataParams const&, impala::TTransmitDataResult*) () #16 0x000000000169a0fe in impala::DataStreamSender::Channel::TransmitDataHelper(impala::TRowBatch const*) () #17 0x0000000001699ccb in impala::DataStreamSender::Channel::TransmitData(int, impala::TRowBatch const*) () #18 0x00000000016a1355 in boost::_mfi::mf2::operator()(impala::DataStreamSender::Channel*, int, impala::TRowBatch const*) const () #19 0x00000000016a0e3a in void boost::_bi::list3, boost::arg<1>, boost::arg<2> >::operator(), boost::_bi::list2 >(boost::_bi::type, boost::_mfi::mf2&, boost::_bi::list2&, int) () #20 0x00000000016a0916 in void boost::_bi::bind_t, boost::_bi::list3, boost::arg<1>, boost::arg<2> > >::operator()(int&, impala::TRowBatch* const&) () #21 0x00000000016a0344 in boost::detail::function::void_function_obj_invoker2, boost::_bi::list3, boost::arg<1>, boost::arg<2> > >, void, int, impala::TRowBatch* const&>::invoke(boost::detail::function::function_buffer&, int, impala::TRowBatch* const&) () #22 0x000000000169fe19 in boost::function2::operator()(int, impala::TRowBatch* const&) const () #23 0x000000000169ef76 in impala::ThreadPool::WorkerThread(int) () #24 0x00000000016a1879 in boost::_mfi::mf1, int>::operator()(impala::ThreadPool*, int) const () #25 0x00000000016a16f1 in void boost::_bi::list2*>, boost::_bi::value >::operator(), int>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1, int>&, boost::_bi::list0&, int) () #26 0x00000000016a1417 in boost::_bi::bind_t, int>, boost::_bi::list2*>, boost::_bi::value > >::operator()() () #27 0x00000000016a0f68 in boost::detail::function::void_function_obj_invoker0, int>, boost::_bi::list2*>, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #28 0x00000000011ca619 in boost::function0::operator()() const () #29 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #30 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #31 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #32 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #33 0x00000000016d8c53 in ?? () #34 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #35 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 24 threads scanner-thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001560222 in impala::BlockingQueue::BlockingPut(impala::RowBatch* const&) () #3 0x000000000155bc28 in impala::ExecNode::RowBatchQueue::AddBatch(impala::RowBatch*) () #4 0x000000000157f03b in impala::HdfsScanNode::AddMaterializedRowBatch(impala::RowBatch*) () #5 0x00000000016802d1 in impala::HdfsScanner::CommitRows(int) () #6 0x00000000015cada3 in impala::HdfsParquetScanner::AssembleRows(int) () #7 0x00000000015ca2b0 in impala::HdfsParquetScanner::ProcessSplit() () #8 0x000000000157fe82 in impala::HdfsScanNode::ScannerThread() () #9 0x0000000001595321 in boost::_mfi::mf0::operator()(impala::HdfsScanNode*) const () #10 0x000000000159491a in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #11 0x0000000001593837 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #12 0x0000000001591e48 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #13 0x00000000011ca619 in boost::function0::operator()() const () #14 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #15 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #16 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #17 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #18 0x00000000016d8c53 in ?? () #19 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #20 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 21 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000169a430 in impala::DataStreamSender::Channel::WaitForRpc() () #3 0x000000000169a772 in impala::DataStreamSender::Channel::SendCurrentBatch() () #4 0x000000000169a4ec in impala::DataStreamSender::Channel::AddRow(impala::TupleRow*) () #5 0x000000000169cf1e in impala::DataStreamSender::Send(impala::RuntimeState*, impala::RowBatch*, bool) () #6 0x00000000015269a1 in impala::PlanFragmentExecutor::OpenInternal() () #7 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #8 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #9 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #10 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #11 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #12 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #13 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #14 0x00000000011ca619 in boost::function0::operator()() const () #15 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #16 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #17 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #18 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #19 0x00000000016d8c53 in ?? () #20 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #21 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 16 threads backend #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000151a645 in impala::DataStreamRecvr::SenderQueue::AddBatch(impala::TRowBatch const&) () #3 0x000000000151c15c in impala::DataStreamRecvr::AddBatch(impala::TRowBatch const&, int) () #4 0x0000000001515a95 in impala::DataStreamMgr::AddData(impala::TUniqueId const&, int, impala::TRowBatch const&, int) () #5 0x000000000125b487 in impala::ImpalaServer::TransmitData(impala::TTransmitDataResult&, impala::TTransmitDataParams const&) () #6 0x000000000126ea55 in impala::ImpalaInternalService::TransmitData(impala::TTransmitDataResult&, impala::TTransmitDataParams const&) () #7 0x000000000144c973 in impala::ImpalaInternalServiceProcessor::process_TransmitData(int, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, void*) () #8 0x000000000144ae94 in impala::ImpalaInternalServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::basic_string, std::allocator > const&, int, void*) () #9 0x00000000012647d2 in apache::thrift::TDispatchProcessor::process(boost::shared_ptr, boost::shared_ptr, void*) () #10 0x0000000001fa7a19 in apache::thrift::server::TThreadedServer::Task::run() () #11 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #12 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #13 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #14 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #15 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #16 0x00000000011ca619 in boost::function0::operator()() const () #17 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #18 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #19 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #20 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #21 0x00000000016d8c53 in ?? () #22 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #23 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 16 threads hdfs-worker #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x00000000013854a6 in impala::BlockingQueue::BlockingGet(impala::HdfsOp*) () #3 0x000000000138415c in impala::ThreadPool::WorkerThread(int) () #4 0x000000000138723f in boost::_mfi::mf1, int>::operator()(impala::ThreadPool*, int) const () #5 0x000000000138710f in void boost::_bi::list2*>, boost::_bi::value >::operator(), int>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1, int>&, boost::_bi::list0&, int) () #6 0x0000000001386d9b in boost::_bi::bind_t, int>, boost::_bi::list2*>, boost::_bi::value > >::operator()() () #7 0x0000000001386a6e in boost::detail::function::void_function_obj_invoker0, int>, boost::_bi::list2*>, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 13 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f7989218243 in os::PlatformEvent::park() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f79891dfaaf in Monitor::IWait(Thread*, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f79891e023e in Monitor::wait(bool, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7988f549e3 in GCTaskManager::get_task(unsigned int) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f7988f560a8 in GCTaskThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #7 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #8 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 10 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162cad4 in impala::PartitionedHashJoinNode::ProcessBuildInput(impala::RuntimeState*, int) () #7 0x000000000162c0fa in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #8 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #10 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #11 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #12 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #13 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #14 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #15 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #16 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #17 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #18 0x00000000011ca619 in boost::function0::operator()() const () #19 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #20 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #21 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #22 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #23 0x00000000016d8c53 in ?? () #24 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #25 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 8 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001565738 in impala::ExchangeNode::Open(impala::RuntimeState*) () #6 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x0000000001648397 in impala::SortNode::Open(impala::RuntimeState*) () #8 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #9 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #10 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #11 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #12 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #13 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #14 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #15 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #16 0x00000000011ca619 in boost::function0::operator()() const () #17 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #18 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #19 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #20 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #21 0x00000000016d8c53 in ?? () #22 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #23 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 6 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001565738 in impala::ExchangeNode::Open(impala::RuntimeState*) () #6 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #8 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #9 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #10 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #11 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #12 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #13 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #14 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #15 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #16 0x00000000011ca619 in boost::function0::operator()() const () #17 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #18 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #19 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #20 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #21 0x00000000016d8c53 in ?? () #22 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #23 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 5 threads cancellation-worker #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x00000000012840a2 in impala::BlockingQueue::BlockingGet(impala::CancellationWork*) () #3 0x0000000001279bb4 in impala::ThreadPool::WorkerThread(int) () #4 0x000000000129bbd9 in boost::_mfi::mf1, int>::operator()(impala::ThreadPool*, int) const () #5 0x000000000129b4ed in void boost::_bi::list2*>, boost::_bi::value >::operator(), int>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1, int>&, boost::_bi::list0&, int) () #6 0x000000000129a44f in boost::_bi::bind_t, int>, boost::_bi::list2*>, boost::_bi::value > >::operator()() () #7 0x0000000001297afa in boost::detail::function::void_function_obj_invoker0, int>, boost::_bi::list2*>, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 5 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000169a430 in impala::DataStreamSender::Channel::WaitForRpc() () #3 0x000000000169a87e in impala::DataStreamSender::Channel::GetSendStatus() () #4 0x0000000001699add in impala::DataStreamSender::Channel::SendBatch(impala::TRowBatch*) () #5 0x000000000169cc12 in impala::DataStreamSender::Send(impala::RuntimeState*, impala::RowBatch*, bool) () #6 0x00000000015269a1 in impala::PlanFragmentExecutor::OpenInternal() () #7 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #8 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #9 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #10 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #11 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #12 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #13 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #14 0x00000000011ca619 in boost::function0::operator()() const () #15 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #16 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #17 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #18 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #19 0x00000000016d8c53 in ?? () #20 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #21 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 4 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000169a430 in impala::DataStreamSender::Channel::WaitForRpc() () #3 0x000000000169a772 in impala::DataStreamSender::Channel::SendCurrentBatch() () #4 0x000000000169ab5d in impala::DataStreamSender::Channel::CloseInternal() () #5 0x000000000169b1b6 in impala::DataStreamSender::Channel::Close(impala::RuntimeState*) () #6 0x000000000169d05d in impala::DataStreamSender::Close(impala::RuntimeState*) () #7 0x000000000152867e in impala::PlanFragmentExecutor::Close() () #8 0x00000000012fbd56 in impala::FragmentMgr::FragmentExecState::Exec() () #9 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #10 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #11 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #12 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #13 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #14 0x00000000011ca619 in boost::function0::operator()() const () #15 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #16 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #17 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #18 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #19 0x00000000016d8c53 in ?? () #20 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #21 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 3 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x00000000016157ea in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x0000000001648397 in impala::SortNode::Open(impala::RuntimeState*) () #8 0x0000000001660d51 in impala::AnalyticEvalNode::Open(impala::RuntimeState*) () #9 0x0000000001646547 in impala::SelectNode::Open(impala::RuntimeState*) () #10 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #11 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #12 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #13 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #14 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #15 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #16 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #17 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #18 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #19 0x00000000011ca619 in boost::function0::operator()() const () #20 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #21 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #22 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #23 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #24 0x00000000016d8c53 in ?? () #25 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #26 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 3 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162cad4 in impala::PartitionedHashJoinNode::ProcessBuildInput(impala::RuntimeState*, int) () #7 0x000000000162c0fa in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #8 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #10 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #11 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #12 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #13 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #14 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #15 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #16 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #17 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #18 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #19 0x00000000011ca619 in boost::function0::operator()() const () #20 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #21 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #22 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #23 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #24 0x00000000016d8c53 in ?? () #25 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #26 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads StatestoreSubscriber #0 0x00000035e240eb5c in recv () from /lib64/libpthread.so.0 #1 0x0000000001f9819d in apache::thrift::transport::TSocket::read(unsigned char*, unsigned int) () #2 0x0000000001fa16d6 in apache::thrift::transport::TBufferedTransport::peek() () #3 0x0000000001fa7a5f in apache::thrift::server::TThreadedServer::Task::run() () #4 0x0000000001192827 in impala::ThriftThread::RunRunnable(boost::shared_ptr, impala::Promise*) () #5 0x0000000001194109 in boost::_mfi::mf2, impala::Promise*>::operator()(impala::ThriftThread*, boost::shared_ptr, impala::Promise*) const () #6 0x0000000001193f62 in void boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> >::operator(), impala::Promise*>, boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf2, impala::Promise*>&, boost::_bi::list0&, int) () #7 0x0000000001193cdf in boost::_bi::bind_t, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #8 0x0000000001193bed in boost::detail::function::void_function_obj_invoker0, impala::Promise*>, boost::_bi::list3, boost::_bi::value >, boost::_bi::value*> > >, void>::invoke(boost::detail::function::function_buffer&) () #9 0x00000000011ca619 in boost::function0::operator()() const () #10 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #11 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #12 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #13 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #14 0x00000000016d8c53 in ?? () #15 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #16 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162cad4 in impala::PartitionedHashJoinNode::ProcessBuildInput(impala::RuntimeState*, int) () #7 0x000000000162c0fa in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #8 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #10 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #11 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #12 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #13 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #14 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #15 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #16 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #17 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #18 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #19 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #20 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #21 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #22 0x00000000011ca619 in boost::function0::operator()() const () #23 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #24 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #25 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #26 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #27 0x00000000016d8c53 in ?? () #28 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #29 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x00000000016157ea in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #8 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #9 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #10 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #11 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #12 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #13 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #14 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #15 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #16 0x00000000011ca619 in boost::function0::operator()() const () #17 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #18 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #19 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #20 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #21 0x00000000016d8c53 in ?? () #22 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #23 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x00000000016157ea in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #8 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #9 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #10 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #11 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #12 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #13 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #14 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #15 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #16 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #17 0x00000000011ca619 in boost::function0::operator()() const () #18 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #19 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #20 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #21 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #22 0x00000000016d8c53 in ?? () #23 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #24 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162d893 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #7 0x000000000162fcf7 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #8 0x00000000016157ea in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #9 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #10 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #11 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #12 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #13 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #14 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #15 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #16 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #17 0x00000000011ca619 in boost::function0::operator()() const () #18 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #19 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #20 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #21 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #22 0x00000000016d8c53 in ?? () #23 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #24 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads Unknown thread #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000001416dc5 in ?? () #2 0x0000000001417198 in ?? () #3 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #4 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 2 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f7989218243 in os::PlatformEvent::park() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f79891dfaaf in Monitor::IWait(Thread*, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f79891e02b6 in Monitor::wait(bool, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7988e49ad8 in CompileQueue::get() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f7988e4dbda in CompileBroker::compiler_thread_loop() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00007f7989357d9f in JavaThread::thread_main_inner() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #7 0x00007f7989357ea5 in JavaThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #8 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #9 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #10 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1ce1be4 in usleep () from /lib64/libc.so.6 #2 0x00000000013e6b59 in impala::SleepForMs(long) () #3 0x00000000013f68f0 in impala::PeriodicCounterUpdater::UpdateLoop() () #4 0x000000000140029d in boost::_mfi::mf0::operator()(impala::PeriodicCounterUpdater*) const () #5 0x000000000140020c in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x00000000014001b9 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x0000000001400178 in boost::detail::thread_data, boost::_bi::list1 > > >::run() () #8 0x00000000016d8c53 in ?? () #9 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #10 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240d930 in sem_wait () from /lib64/libpthread.so.0 #1 0x00007f798921d5ca in check_pending_signals(bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f7989216835 in signal_thread_entry(JavaThread*, Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f7989357d9f in JavaThread::thread_main_inner() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7989357ea5 in JavaThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #7 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f7989218243 in os::PlatformEvent::park() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f798920cadd in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f798907c378 in JVM_MonitorWait () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7983f6e7f8 in ?? () #5 0x0000000000000000 in ?? () 1 threads Unknown thread #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f798921d114 in Parker::park(bool, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f798937ba05 in Unsafe_Park () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f7983f6e7f8 in ?? () #4 0x00007f7983f62058 in ?? () #5 0x0000000000000000 in ?? () 1 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f7989218243 in os::PlatformEvent::park() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f79891dfaaf in Monitor::IWait(Thread*, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f79891e023e in Monitor::wait(bool, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f79892ae388 in ServiceThread::service_thread_entry(JavaThread*, Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f7989357d9f in JavaThread::thread_main_inner() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00007f7989357ea5 in JavaThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #7 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #8 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #9 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001565738 in impala::ExchangeNode::Open(impala::RuntimeState*) () #6 0x000000000162c06f in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #7 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #8 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #9 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #10 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #11 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #12 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #13 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #14 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #15 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #16 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #17 0x00000000011ca619 in boost::function0::operator()() const () #18 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #19 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #20 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #21 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #22 0x00000000016d8c53 in ?? () #23 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #24 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads log-flush-thread #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1cac8b0 in sleep () from /lib64/libc.so.6 #2 0x0000000001254fdc in impala::ImpalaServer::LogFileFlushThread() () #3 0x000000000129b731 in boost::_mfi::mf0::operator()(impala::ImpalaServer*) const () #4 0x000000000129a548 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #5 0x0000000001297bbb in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #6 0x0000000001294131 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #7 0x00000000011ca619 in boost::function0::operator()() const () #8 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #9 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #10 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #11 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #12 0x00000000016d8c53 in ?? () #13 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #14 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads recovery-mode-thread #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1ce1be4 in usleep () from /lib64/libc.so.6 #2 0x00000000013e6b59 in impala::SleepForMs(long) () #3 0x000000000134e46b in impala::StatestoreSubscriber::RecoveryModeChecker() () #4 0x000000000135b433 in boost::_mfi::mf0::operator()(impala::StatestoreSubscriber*) const () #5 0x000000000135ae40 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x000000000135a941 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x0000000001359e67 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads maintenance-thread #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1cac8b0 in sleep () from /lib64/libc.so.6 #2 0x0000000000f12ac8 in ?? () #3 0x0000000000f14c1c in boost::detail::function::void_function_invoker0::invoke(boost::detail::function::function_buffer&) () #4 0x00000000011ca619 in boost::function0::operator()() const () #5 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #6 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #7 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #8 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #9 0x00000000016d8c53 in ?? () #10 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #11 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x00007f795f6e847a in Java_org_apache_hadoop_net_unix_DomainSocketWatcher_doPoll0 () from /data/1/parcels/CDH-5.5.0-1.cdh5.5.0.p0.1003/lib/hadoop/lib/native/libhadoop.so.1.0.0 #2 0x00007f7983f6e7f8 in ?? () #3 0x000000000708a9e8 in ?? () #4 0x00000000066d4f00 in ?? () #5 0x00007f795fc31640 in ?? () #6 0x0000000000000000 in ?? () 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x00000000016157ea in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #7 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #8 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #10 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #11 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #12 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #13 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #14 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #15 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #16 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #17 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #18 0x00000000011ca619 in boost::function0::operator()() const () #19 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #20 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #21 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #22 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #23 0x00000000016d8c53 in ?? () #24 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #25 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f798921d8ef in os::PlatformEvent::park(long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f79891dfd5e in Monitor::IWait(Thread*, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f79891e023e in Monitor::wait(bool, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f79893a5169 in VMThread::loop() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f79893a5470 in VMThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #7 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #8 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f7989218243 in os::PlatformEvent::park() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f798920cadd in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f798907c378 in JVM_MonitorWait () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7983f6e7f8 in ?? () #5 0x00007f79671ea7a0 in ?? () #6 0x00007f7983f62058 in ?? () #7 0x000000050fdfc710 in ?? () #8 0x00007f79671ea748 in ?? () #9 0x0000000000000000 in ?? () 1 threads supervise-beeswax-frontend #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x0000000001f9edfa in apache::thrift::transport::TServerSocket::acceptImpl() () #2 0x0000000001fa2083 in apache::thrift::server::TThreadPoolServer::serve() () #3 0x000000000118783a in impala::ThriftServer::ThriftServerEventProcessor::Supervise() () #4 0x0000000001190e25 in boost::_mfi::mf0::operator()(impala::ThriftServer::ThriftServerEventProcessor*) const () #5 0x0000000001190b92 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x0000000001190733 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x000000000118fe57 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162cad4 in impala::PartitionedHashJoinNode::ProcessBuildInput(impala::RuntimeState*, int) () #7 0x000000000162c0fa in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #8 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #10 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #11 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #12 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #13 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #14 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #15 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #16 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #17 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #18 0x00000000011ca619 in boost::function0::operator()() const () #19 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #20 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #21 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #22 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #23 0x00000000016d8c53 in ?? () #24 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #25 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads audit-event-log-flush #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1cac8b0 in sleep () from /lib64/libc.so.6 #2 0x0000000001255025 in impala::ImpalaServer::AuditEventLoggerFlushThread() () #3 0x000000000129b731 in boost::_mfi::mf0::operator()(impala::ImpalaServer*) const () #4 0x000000000129a548 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #5 0x0000000001297bbb in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #6 0x0000000001294131 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #7 0x00000000011ca619 in boost::function0::operator()() const () #8 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #9 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #10 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #11 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #12 0x00000000016d8c53 in ?? () #13 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #14 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f798921d8ef in os::PlatformEvent::park(long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f79891dfd5e in Monitor::IWait(Thread*, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f79891e023e in Monitor::wait(bool, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7989353dc1 in WatcherThread::sleep() const () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #5 0x00007f79893542de in WatcherThread::run() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #6 0x00007f798921e988 in java_start(Thread*) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #7 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #8 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads supervise-backend #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x0000000001f9edfa in apache::thrift::transport::TServerSocket::acceptImpl() () #2 0x0000000001fa50a6 in apache::thrift::server::TThreadedServer::serve() () #3 0x000000000118783a in impala::ThriftServer::ThriftServerEventProcessor::Supervise() () #4 0x0000000001190e25 in boost::_mfi::mf0::operator()(impala::ThriftServer::ThriftServerEventProcessor*) const () #5 0x0000000001190b92 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x0000000001190733 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x000000000118fe57 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162cad4 in impala::PartitionedHashJoinNode::ProcessBuildInput(impala::RuntimeState*, int) () #7 0x000000000162c0fa in impala::PartitionedHashJoinNode::ConstructBuildSide(impala::RuntimeState*) () #8 0x000000000166e992 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #9 0x000000000166ea42 in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #10 0x000000000164d269 in impala::TopNNode::Open(impala::RuntimeState*) () #11 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #12 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #13 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #14 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #15 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #16 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #17 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #18 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #19 0x00000000011ca619 in boost::function0::operator()() const () #20 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #21 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #22 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #23 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #24 0x00000000016d8c53 in ?? () #25 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #26 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x00000000016d9f2c in boost::thread::join_noexcept() () #3 0x000000000118a682 in boost::thread::join() () #4 0x000000000118af5e in impala::Thread::Join() const () #5 0x0000000001189bfc in impala::ThriftServer::Join() () #6 0x0000000000eda0f1 in main () 1 threads supervise-StatestoreSubscriber #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x0000000001f9edfa in apache::thrift::transport::TServerSocket::acceptImpl() () #2 0x0000000001fa50a6 in apache::thrift::server::TThreadedServer::serve() () #3 0x000000000118783a in impala::ThriftServer::ThriftServerEventProcessor::Supervise() () #4 0x0000000001190e25 in boost::_mfi::mf0::operator()(impala::ThriftServer::ThriftServerEventProcessor*) const () #5 0x0000000001190b92 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x0000000001190733 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x000000000118fe57 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x0000000001417786 in ?? () #2 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #3 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads query-expirer #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1ce1be4 in usleep () from /lib64/libc.so.6 #2 0x00000000013e6b59 in impala::SleepForMs(long) () #3 0x00000000012614f7 in impala::ImpalaServer::ExpireQueries() () #4 0x000000000129b731 in boost::_mfi::mf0::operator()(impala::ImpalaServer*) const () #5 0x000000000129a548 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x0000000001297bbb in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x0000000001294131 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads supervise-hiveserver2-frontend #0 0x00000035e1cdf0d3 in poll () from /lib64/libc.so.6 #1 0x0000000001f9edfa in apache::thrift::transport::TServerSocket::acceptImpl() () #2 0x0000000001fa2083 in apache::thrift::server::TThreadPoolServer::serve() () #3 0x000000000118783a in impala::ThriftServer::ThriftServerEventProcessor::Supervise() () #4 0x0000000001190e25 in boost::_mfi::mf0::operator()(impala::ThriftServer::ThriftServerEventProcessor*) const () #5 0x0000000001190b92 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #6 0x0000000001190733 in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #7 0x000000000118fe57 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #8 0x00000000011ca619 in boost::function0::operator()() const () #9 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #10 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #11 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #12 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #13 0x00000000016d8c53 in ?? () #14 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #15 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x0000000001519e08 in impala::DataStreamRecvr::SenderQueue::GetBatch(impala::RowBatch**) () #3 0x000000000151c6b5 in impala::DataStreamRecvr::GetBatch(impala::RowBatch**) () #4 0x0000000001565a5a in impala::ExchangeNode::FillInputRowBatch(impala::RuntimeState*) () #5 0x0000000001566209 in impala::ExchangeNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #6 0x000000000162d893 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #7 0x000000000162fcf7 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #8 0x000000000162d893 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #9 0x000000000162fcf7 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #10 0x000000000162d893 in impala::PartitionedHashJoinNode::NextProbeRowBatch(impala::RuntimeState*, impala::RowBatch*) () #11 0x000000000162fcf7 in impala::PartitionedHashJoinNode::GetNext(impala::RuntimeState*, impala::RowBatch*, bool*) () #12 0x000000000166eb1f in impala::BlockingJoinNode::Open(impala::RuntimeState*) () #13 0x00000000016155df in impala::PartitionedAggregationNode::Open(impala::RuntimeState*) () #14 0x0000000001526376 in impala::PlanFragmentExecutor::OpenInternal() () #15 0x0000000001526222 in impala::PlanFragmentExecutor::Open() () #16 0x00000000012fbd38 in impala::FragmentMgr::FragmentExecState::Exec() () #17 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #18 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #19 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #20 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #21 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #22 0x00000000011ca619 in boost::function0::operator()() const () #23 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #24 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #25 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #26 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #27 0x00000000016d8c53 in ?? () #28 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #29 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f798921d228 in Parker::park(bool, long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f798937ba05 in Unsafe_Park () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f7983f6e7f8 in ?? () #4 0x00007f74527964d0 in ?? () #5 0x000000050a0f6b59 in ?? () #6 0x00007f7452796538 in ?? () #7 0x000000050a0f7370 in ?? () #8 0x0000000000000000 in ?? () 1 threads lineage-log-flush #0 0x00000035e1caca3d in nanosleep () from /lib64/libc.so.6 #1 0x00000035e1cac8b0 in sleep () from /lib64/libc.so.6 #2 0x00000000012551df in impala::ImpalaServer::LineageLoggerFlushThread() () #3 0x000000000129b731 in boost::_mfi::mf0::operator()(impala::ImpalaServer*) const () #4 0x000000000129a548 in void boost::_bi::list1 >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf0&, boost::_bi::list0&, int) () #5 0x0000000001297bbb in boost::_bi::bind_t, boost::_bi::list1 > >::operator()() () #6 0x0000000001294131 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke(boost::detail::function::function_buffer&) () #7 0x00000000011ca619 in boost::function0::operator()() const () #8 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #9 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #10 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #11 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #12 0x00000000016d8c53 in ?? () #13 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #14 0x00000035e1ce88fd in clone () from /lib64/libc.so.6 1 threads Unknown thread #0 0x00000035e240b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f798921d8ef in os::PlatformEvent::park(long) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #2 0x00007f798921e3d4 in os::sleep(Thread*, long, bool) () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #3 0x00007f7989084892 in JVM_Sleep () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #4 0x00007f7984560d11 in ?? () #5 0x00007f79554a67c0 in ?? () #6 0x00000035e3003e46 in clock_gettime () from /lib64/librt.so.1 #7 0x00007f79892180cf in os::javaTimeNanos() () from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so #8 0x00007f798466b430 in ?? () #9 0x000000050fdfd270 in ?? () #10 0x00007f7983f62058 in ?? () #11 0x0000000000000bb8 in ?? () #12 0x0000000000000000 in ?? () 1 threads exec-plan-fragment #0 0x00000035e240b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000115e82b in boost::condition_variable::wait(boost::unique_lock&) () #2 0x000000000169a430 in impala::DataStreamSender::Channel::WaitForRpc() () #3 0x000000000169a87e in impala::DataStreamSender::Channel::GetSendStatus() () #4 0x000000000169abf4 in impala::DataStreamSender::Channel::CloseInternal() () #5 0x000000000169b1b6 in impala::DataStreamSender::Channel::Close(impala::RuntimeState*) () #6 0x000000000169d05d in impala::DataStreamSender::Close(impala::RuntimeState*) () #7 0x000000000152867e in impala::PlanFragmentExecutor::Close() () #8 0x00000000012fbd56 in impala::FragmentMgr::FragmentExecState::Exec() () #9 0x00000000012f4356 in impala::FragmentMgr::FragmentExecThread(impala::FragmentMgr::FragmentExecState*) () #10 0x00000000012f7d46 in boost::_mfi::mf1::operator()(impala::FragmentMgr*, impala::FragmentMgr::FragmentExecState*) const () #11 0x00000000012f7aff in void boost::_bi::list2, boost::_bi::value >::operator(), boost::_bi::list0>(boost::_bi::type, boost::_mfi::mf1&, boost::_bi::list0&, int) () #12 0x00000000012f73e7 in boost::_bi::bind_t, boost::_bi::list2, boost::_bi::value > >::operator()() () #13 0x00000000012f6d35 in boost::detail::function::void_function_obj_invoker0, boost::_bi::list2, boost::_bi::value > >, void>::invoke(boost::detail::function::function_buffer&) () #14 0x00000000011ca619 in boost::function0::operator()() const () #15 0x00000000013dd8f2 in impala::Thread::SuperviseThread(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*) () #16 0x00000000013e6a7a in void boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> >::operator(), std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0>(boost::_bi::type, void (*&)(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list0&, int) () #17 0x00000000013e69c1 in boost::_bi::bind_t, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > >::operator()() () #18 0x00000000013e6980 in boost::detail::thread_data, std::allocator > const&, std::basic_string, std::allocator > const&, boost::function, impala::Promise*), boost::_bi::list4, std::allocator > >, boost::_bi::value, std::allocator > >, boost::_bi::value >, boost::_bi::value*> > > >::run() () #19 0x00000000016d8c53 in ?? () #20 0x00000035e24079d1 in start_thread () from /lib64/libpthread.so.0 #21 0x00000035e1ce88fd in clone () from /lib64/libc.so.6