Uploaded image for project: 'Qpid Proton'
  1. Qpid Proton
  2. PROTON-1591

[proton-cpp] Scheduling task from a scheduled task will deadlock

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: proton-c-0.18.0
    • Component/s: cpp-binding
    • Labels:
      None
    • Environment:

      Description

      Modify the cpp/simple_send.cpp example the following way

      diff --git a/examples/cpp/simple_send.cpp b/examples/cpp/simple_send.cpp
      index a4c2272d..92e60ee0 100644
      --- a/examples/cpp/simple_send.cpp
      +++ b/examples/cpp/simple_send.cpp
      @@ -29,9 +29,11 @@
       #include <proton/messaging_handler.hpp>
       #include <proton/tracker.hpp>
       #include <proton/types.hpp>
      +#include <proton/work_queue.hpp>
       
       #include <iostream>
       #include <map>
      +#include <functional>
       
       #include "fake_cpp11.hpp"
       
      @@ -45,15 +47,28 @@ class simple_send : public proton::messaging_handler {
           int confirmed;
           int total;
       
      +    std::function<void()> callback;
      +
         public:
           simple_send(const std::string &s, const std::string &u, const std::string &p, int c) :
      -        url(s), user(u), password(p), sent(0), confirmed(0), total(c) {}
      +        url(s), user(u), password(p), sent(0), confirmed(0), total(c) {
      +        callback = [this]() {
      +            std::cout << "Entering callback" << std::endl;
      +//            TODO: uncomment one of the two commands below
      +//****************************************************************************************************
      +//            sender.container().stop();
      +//            sender.container().schedule(1 * proton::duration::SECOND, proton::work(callback));
      +//****************************************************************************************************
      +            std::cout << "Leaving callback" << std::endl;
      +        };
      +    }
       
           void on_container_start(proton::container &c) OVERRIDE {
               proton::connection_options co;
               if (!user.empty()) co.user(user);
               if (!password.empty()) co.password(password);
               sender = c.open_sender(url, co);
      +        c.schedule(1 * proton::duration::SECOND, proton::work(callback));
           }
       
           void on_sendable(proton::sender &s) OVERRIDE {
      

      Now uncomment one of the two commented out commands. Either to stop the container from the inside scheduled task, or to schedule new task from inside the scheduled task. When executed, the program will deadlock.

      Program received signal SIGINT, Interrupt.
      0x00007ffff68ef6dc in __lll_lock_wait () from /nix/store/l48biijfr1j6d5kdg911051x2phfjrz7-glibc-2.25/lib/libpthread.so.0
      (gdb) bt
      #0  0x00007ffff68ef6dc in __lll_lock_wait () from /nix/store/l48biijfr1j6d5kdg911051x2phfjrz7-glibc-2.25/lib/libpthread.so.0
      #1  0x00007ffff68e88e5 in pthread_mutex_lock () from /nix/store/l48biijfr1j6d5kdg911051x2phfjrz7-glibc-2.25/lib/libpthread.so.0
      #2  0x00007ffff7baf1e8 in __gthread_mutex_lock (__mutex=0x61e148) at /nix/store/pdidaf83cvkrgx8xjgjdnl5m1naqjbfk-gcc-7.1.0/include/c++/7.1.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748
      #3  std::mutex::lock (this=0x61e148) at /nix/store/pdidaf83cvkrgx8xjgjdnl5m1naqjbfk-gcc-7.1.0/include/c++/7.1.0/bits/std_mutex.h:103
      #4  std::lock_guard<std::mutex>::lock_guard (__m=..., this=<synthetic pointer>) at /nix/store/pdidaf83cvkrgx8xjgjdnl5m1naqjbfk-gcc-7.1.0/include/c++/7.1.0/bits/std_mutex.h:162
      #5  proton::container::impl::schedule (this=this@entry=0x61e140, delay=..., delay@entry=..., f=...) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:399
      #6  0x00007ffff7baccc8 in proton::container::schedule (this=this@entry=0x7fffffffc668, d=..., f=...) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/container.cpp:109
      #7  0x00000000004065f9 in simple_send::simple_send(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)::{lambda()#1}::operator()() const (__closure=<optimized out>) at /home/jdanek/Work/repos/qpid-proton/examples/cpp/simple_send.cpp:58
      #8  std::_Function_handler<void (), simple_send::simple_send(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)::{lambda()#1}>::_M_invoke(std::_Any_data const&) (__functor=...) at /nix/store/pdidaf83cvkrgx8xjgjdnl5m1naqjbfk-gcc-7.1.0/include/c++/7.1.0/bits/std_function.h:316
      #9  0x00007ffff7baf706 in std::function<void ()>::operator()() const (this=<optimized out>) at /nix/store/pdidaf83cvkrgx8xjgjdnl5m1naqjbfk-gcc-7.1.0/include/c++/7.1.0/bits/std_function.h:706
      #10 proton::work::operator() (this=<optimized out>) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/include/proton/work_queue.hpp:59
      #11 proton::container::impl::run_timer_jobs (this=this@entry=0x61e140) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:439
      #12 0x00007ffff7bafe10 in proton::container::impl::handle (this=this@entry=0x61e140, event=0x625590) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:478
      #13 0x00007ffff7baffeb in proton::container::impl::thread (this=this@entry=0x61e140) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:613
      #14 0x00007ffff7bb0393 in proton::container::impl::run (this=0x61e140, threads=threads@entry=1) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:651
      #15 0x00007ffff7bac74d in proton::container::run (this=this@entry=0x7fffffffc668) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/container.cpp:83
      #16 0x0000000000404d2f in main (argc=<optimized out>, argv=<optimized out>) at /home/jdanek/Work/repos/qpid-proton/examples/cpp/simple_send.cpp:115
      #17 0x00007ffff6b1d530 in __libc_start_main () from /nix/store/l48biijfr1j6d5kdg911051x2phfjrz7-glibc-2.25/lib/libc.so.6
      #18 0x00000000004051aa in _start () at ../sysdeps/x86_64/start.S:120
      

      The lock was first acquired here

      (gdb) frame 12
      #12 0x00007ffff7bafe10 in proton::container::impl::handle (this=this@entry=0x61e140, event=0x625590) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:478
      478	            run_timer_jobs();
      (gdb) l
      473	
      474	    case PN_PROACTOR_TIMEOUT: {
      475	        GUARD(lock_);
      476	        // Can get an immediate timeout, if we have a container event loop inject
      477	        if  ( deferred_.size()>0 ) {
      478	            run_timer_jobs();
      479	        }
      480	
      481	        // Run every container event loop job
      482	        // This is not at all efficient and single threads all these jobs, but it does correctly
      

      and the thread tries to reacquire it again here (this is the schedule task from scheduled task case)

      (gdb) frame 5
      #5  proton::container::impl::schedule (this=this@entry=0x61e140, delay=..., delay@entry=..., f=...) at /home/jdanek/Work/repos/qpid-proton/proton-c/bindings/cpp/src/proactor_container_impl.cpp:399
      399	    GUARD(lock_);
      (gdb) l
      394	    lc.listen_handler_ = &lh;
      395	    return proton::listener(listener);
      396	}
      397	
      398	void container::impl::schedule(duration delay, work f) {
      399	    GUARD(lock_);
      400	    timestamp now = timestamp::now();
      401	
      402	    // Record timeout; Add callback to timeout sorted list
      403	    scheduled s = {now+delay, f};
      

      This reproducer is based on larger application which was using the previous version of the task scheduling API and which was now updated to compile with Proton 0.18.

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                astitcher Andrew Stitcher
                Reporter:
                jdanek Jiri Daněk
              • Votes:
                0 Vote for this issue
                Watchers:
                3 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: