[MESOS-392] FaultTolerance SchedulerExit test hangs

• Type: Bug
• Status: Resolved
• Priority: Blocker
• Resolution: Fixed
• Fix Version/s: 0.13.0
• Assignee: Benjamin Hindman
• Reporter: Vinod Kone
• Created: 14/Mar/13 02:01
• Updated: 10/Jun/13 18:07
• Resolved: 10/Jun/13 18:07

Description

This is gdb trace of interesting threads in the test process.

Thread 40 (Thread 0x411bf940 (LWP 41716)):
#0 0x000000316120d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003161208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003161208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004c632b in testing::internal::MutexBase::Lock (this=0xbeab40) at ../third_party/libprocess/third_party/gmock-1.6.0/gtest/include/gtest/internal/gtest-port.h:1208
#4 0x00000000007e0355 in testing::internal::GTestMutexLock::GTestMutexLock(testing::internal::MutexBase*) ()
#5 0x00000000007e43bc in testing::internal::UntypedFunctionMockerBase::SetOwnerAndName(void const*, char const*) ()
#6 0x00000000004c5888 in mesos::internal::tests::TestsFilter::filter (this=0x80, gmock_a1=...) at ../../src/tests/filter.hpp:87
#7 0x00007f603a7fda09 in process::ProcessBase::enqueue(process::Event*, bool)::FilterVisitor::visit(process::ExitedEvent const&) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#8 0x00007f603a813928 in process::ExitedEvent::visit(process::EventVisitor*) const () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#9 0x00007f603a8002b5 in process::ProcessBase::enqueue(process::Event*, bool) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#10 0x00007f603a804c52 in process::SocketManager::exited(process::ProcessBase*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#11 0x00007f603a805202 in process::ProcessManager::cleanup(process::ProcessBase*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#12 0x00007f603a807c05 in process::ProcessManager::resume(process::ProcessBase*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#13 0x00007f603a8082b2 in process::schedule(void*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#14 0x000000316120673d in start_thread () from /lib64/libpthread.so.0
#15 0x00000031602d3f6d in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x48afd940 (LWP 41732)):

#0 0x000000316120d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003161208e35 in _L_lock_1127 () from /lib64/libpthread.so.0
#2 0x0000003161208d33 in pthread_mutex_lock () from /lib64/libpthread.so.0

#3 0x00007f603a85474f in Synchronizable::acquire() () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#4 0x00007f603a854779 in Synchronized::Synchronized(Synchronizable*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#5 0x00007f603a7fee1d in process::ProcessManager::use(process::UPID const&) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#6 0x00007f603a8084ee in process::ProcessManager::deliver(process::UPID const&, process::Event*, process::ProcessBase*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#7 0x00007f603a80b1e4 in process::internal::dispatch(process::UPID const&, std::tr1::shared_ptr<std::tr1::function<void ()(process::ProcessBase*)> > const&) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#8 0x00007f603a847a07 in process::Timer::operator()() const () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#9 0x00007f603a809612 in process::handle_timeouts(ev_loop*, ev_timer*, int) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#10 0x00007f603a993a23 in ev_invoke_pending (loop=0x7f6034004c40) at ev.c:1971
#11 0x00007f603a998e58 in ev_loop (loop=0x7f603ae24f20, flags=0) at ev.c:2333
#12 0x00007f603a803c7a in process::serve(void*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#13 0x000000316120673d in start_thread () from /lib64/libpthread.so.0
#14 0x00000031602d3f6d in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f60397b7720 (LWP 41670)):
#0 0x000000316120d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003161208e35 in _L_lock_1127 () from /lib64/libpthread.so.0
#2 0x0000003161208d33 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f603a85474f in Synchronizable::acquire() () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#4 0x00007f603a854779 in Synchronized::Synchronized(Synchronizable*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#5 0x00007f603a7fee1d in process::ProcessManager::use(process::UPID const&) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#6 0x00007f603a805ada in process::ProcessManager::terminate(process::UPID const&, bool, process::ProcessBase*) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#7 0x00007f603a805d82 in process::terminate(process::UPID const&, bool) () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#8 0x00007f603a86cf4d in process::Latch::~Latch() () from /home/vinod/mesos/build/src/.libs/libmesos-0.13.0.so
#9 0x00000000004f6196 in process::Future<Nothing>::cleanup (this=0x7f602c17a188) at ../../third_party/libprocess/include/process/future.hpp:1030
#10 0x00000000004f6364 in Impl<process::Future<Nothing> ()(const Option<mesos::internal::slave::state::SlaveState>&)>::~Impl(void) (this=0x7f602c17a180, __in_chrg=<value optimized out>)
at ../third_party/libprocess/third_party/gmock-1.6.0/include/gmock/gmock-actions.h:478
#11 0x00000000004daa51 in depart (this=0x25f55a0, __in_chrg=<value optimized out>) at ../third_party/libprocess/third_party/gmock-1.6.0/gtest/include/gtest/internal/gtest-linked_ptr.h:195
#12 ~linked_ptr (this=0x25f55a0, __in_chrg=<value optimized out>) at ../third_party/libprocess/third_party/gmock-1.6.0/gtest/include/gtest/internal/gtest-linked_ptr.h:143
#13 ~Action (this=0x25f55a0, __in_chrg=<value optimized out>) at ../third_party/libprocess/third_party/gmock-1.6.0/include/gmock/gmock-actions.h:278
#14 testing::internal::TypedExpectation<process::Future<Nothing> ()(const Option<mesos::internal::slave::state::SlaveState>&)>::~TypedExpectation(void) (this=0x25f55a0, __in_chrg=<value optimized out>)
at ../third_party/libprocess/third_party/gmock-1.6.0/include/gmock/gmock-spec-builders.h:889
#15 0x00000000007ec085 in testing::internal::linked_ptr<testing::internal::ExpectationBase>::depart() ()
#16 0x00000000007ec09d in testing::internal::linked_ptr<testing::internal::ExpectationBase>::~linked_ptr() ()
#17 0x00000000007ec247 in void std::_Destroy<testing::internal::linked_ptr<testing::internal::ExpectationBase> >(testing::internal::linked_ptr<testing::internal::ExpectationBase>*) ()
#18 0x00000000007ec265 in void std::__destroy_aux<testing::internal::linked_ptr<testing::internal::ExpectationBase>>(testing::internal::linked_ptr<testing::internal::ExpectationBase>, testing::internal::linked_ptr<testing::internal::ExpectationBase>*, __false_type) ()
#19 0x00000000007ec29a in void std::_Destroy<testing::internal::linked_ptr<testing::internal::ExpectationBase>>(testing::internal::linked_ptr<testing::internal::ExpectationBase>, testing::internal::linked_ptr<testing::internal::ExpectationBase>*) ()
#20 0x00000000007ec2bd in void std::_Destroy<testing::internal::linked_ptr<testing::internal::ExpectationBase>, testing::internal::linked_ptr<testing::internal::ExpectationBase> >(testing::internal::linked_ptr<testing::internal::ExpectationBase>, testing::internal::linked_ptr<testing::internal::ExpectationBase>*, std::allocator<testing::internal::linked_ptr<testing::internal::ExpectationBase> >) ()
#21 0x00000000007ec2fa in std::vector<testing::internal::linked_ptr<testing::internal::ExpectationBase>, std::allocator<testing::internal::linked_ptr<testing::internal::ExpectationBase> > >::clear() ()
#22 0x00000000007e7238 in testing::internal::UntypedFunctionMockerBase::VerifyAndClearExpectationsLocked() ()
#23 0x0000000000504b94 in testing::internal::FunctionMockerBase<process::Future<Nothing> ()(const Option<mesos::internal::slave::state::SlaveState>&)>::~FunctionMockerBase(void) (this=0x7fffc9b57060, __in_chrg=<value optimized out>)
at ../third_party/libprocess/third_party/gmock-1.6.0/include/gmock/gmock-spec-builders.h:1399
#24 0x000000000054944e in ~FunctionMocker (this=0x7fffc9b56f40, __in_chrg=<value optimized out>, __vtt_parm=<value optimized out>) at ../third_party/libprocess/third_party/gmock-1.6.0/include/gmock/gmock-spec-builders.h:89
#25 mesos::internal::tests::TestingIsolationModule::~TestingIsolationModule (this=0x7fffc9b56f40, __in_chrg=<value optimized out>, __vtt_parm=<value optimized out>) at ../../src/tests/utils.hpp:781
#26 0x00000000006337dd in FaultToleranceTest_SchedulerExit_Test::TestBody (this=0x2628160) at ../../src/tests/fault_tolerance_tests.cpp:935
#27 0x00000000007cebc0 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::)(), char const) ()
#28 0x00000000007e13b6 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::)(), char const) ()
#29 0x00000000007c64f3 in testing::Test::Run() ()
#30 0x00000000007cd27c in testing::TestInfo::Run() ()
#31 0x00000000007cd3cf in testing::TestCase::Run() ()
#32 0x00000000007cd6d2 in testing::internal::UnitTestImpl::RunAllTests() ()
#33 0x00000000007cf1de in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::)(), char const) ()
#34 0x00000000007e0e30 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::)(), char const) ()
#35 0x00000000007c5c80 in testing::UnitTest::Run() ()
--Type <return> to continue, or q <return> to quit--
#36 0x000000000047ffe9 in main (argc=1, argv=<value optimized out>) at ../../src/tests/main.cpp:103