Uploaded image for project: 'Qpid Dispatch'
  1. Qpid Dispatch
  2. DISPATCH-2185

HTTP/1.x crash on message content sys_mutex_lock fail

Attach filesAttach ScreenshotVotersWatch issueWatchersCreate sub-taskLinkCloneUpdate Comment AuthorReplace String in CommentUpdate Comment VisibilityDelete Comments
    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical
    • Resolution: Fixed
    • Affects Version/s: 1.16.0
    • Fix Version/s: 1.17.0
    • Component/s: Protocol Adaptors
    • Labels:
      None

      Description

      Hit during router shutdown after Http1AdaptorBadEndpointsTest:

      {{72: ====================================================================== }}
      {{72: ERROR: tearDownClass (system_tests_http1_adaptor.Http1AdaptorBadEndpointsTest) }}
      {{72: ---------------------------------------------------------------------- }}
      {{72: Traceback (most recent call last): }}
      {{72: File "/home/kgiusti/work/dispatch/qpid-dispatch/tests/system_test.py", line 865, in tearDownClass }}
      {{72: cls.tester.teardown() }}
      {{72: File "/home/kgiusti/work/dispatch/qpid-dispatch/tests/system_test.py", line 808, in teardown }}
      {{72: raise RuntimeError("Errors during teardown: \n\n%s" % "\n\n".join([str(e) for e in errors])) }}
      {{72: RuntimeError: Errors during teardown: }}
      {{72: }}
      {{72: Process 1257318 error: exit code -6, expected -1 }}
      {{72: qdrouterd -c TestBadEndpoints.conf -I /home/kgiusti/work/dispatch/qpid-dispatch/python }}
      {{72: /home/kgiusti/work/dispatch/qpid-dispatch/BUILD/tests/system_test.dir/system_tests_http1_adaptor/Http1AdaptorBadEndpointsTest/setUpClass/TestBadEndpoints-1.cmd }}
      {{72: >>>> }}
      {{72: qdrouterd: /home/kgiusti/work/dispatch/qpid-dispatch/src/posix/threading.c:58: sys_mutex_lock: Assertion `result == 0' failed. }}
      {{72: <<<< }}

       

      GDB thread backtrace:

      {{Thread 5 (Thread 0x7f7f9615f700 (LWP 1257324)): }}
      {{#0 futex_wait_cancelable (private=0, expected=0, futex_word=0x9c09ac) at ../sysdeps/nptl/futex-internal.h:183 }}
      {{#1 __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x9c0a40, cond=0x9c0980) at pthread_cond_wait.c:508 }}
      {{#2 __pthread_cond_wait (cond=0x9c0980, mutex=0x9c0a40) at pthread_cond_wait.c:638 }}
      {{#3 0x00007f7fa47a7a89 in sys_cond_wait (cond=<optimized out>, held_mutex=<optimized out>) at /home/kgiusti/work/dispatch/qpid-dispatch/src/posix/threading.c:92 }}
      {{#4 0x00007f7fa47c4dd7 in router_core_thread (arg=0x9c04c0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/router_core/router_core_thread.c:221 }}
      {{#5 0x00007f7fa46b5432 in start_thread (arg=<optimized out>) at pthread_create.c:477 }}
      {{#6 0x00007f7fa41f96d3 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 }}

      {{Thread 4 (Thread 0x7f7f9578e700 (LWP 1257331)): }}
      {{#0 0x00007f7fa41f9a1e in epoll_wait (epfd=4, events=0x89c9f0, maxevents=16, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30 }}
      {{#1 0x00007f7fa4735e37 in poller_do_epoll (can_block=true, ts=0x8eb2a0, p=0x95e4a0) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:2496 }}
      {{#2 next_event_batch (p=0x95e4a0, can_block=true) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:2442 }}
      {{#3 0x00007f7fa47db54f in thread_run (arg=0x8d89f0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1106 }}
      {{#4 0x00007f7fa46b5432 in start_thread (arg=<optimized out>) at pthread_create.c:477 }}
      {{#5 0x00007f7fa41f96d3 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 }}

      {{Thread 3 (Thread 0x7f7f94dfd700 (LWP 1257332)): }}
      {{#0 0x00007f7fa470cb97 in pn_transport_pending (transport=<optimized out>) at /home/kgiusti/work/proton/qpid-proton/c/src/core/transport.c:2838 }}
      {{#1 0x00007f7fa4707692 in pn_connection_driver_write_buffer (d=d@entry=0xeddc10) at /home/kgiusti/work/proton/qpid-proton/c/src/core/connection_driver.c:112 }}
      {{#2 0x00007f7fa4731030 in ensure_wbuf (pc=0xeddad0) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:1076 }}
      {{#3 write_flush (pc=pc@entry=0xeddad0) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:1076 }}
      {{#4 0x00007f7fa47325be in pconnection_batch_next (batch=0xeddc08) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:892 }}
      {{#5 0x00007f7fa47db55a in thread_run (arg=0x8d89f0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1111 }}
      {{#6 0x00007f7fa46b5432 in start_thread (arg=<optimized out>) at pthread_create.c:477 }}
      {{#7 0x00007f7fa41f96d3 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 }}

      {{Thread 2 (Thread 0x7f7f9446c700 (LWP 1257333)): }}
      {{#0 futex_wait_cancelable (private=0, expected=0, futex_word=0xa3f734) at ../sysdeps/nptl/futex-internal.h:183 }}
      {{#1 __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0xa3f6e0, cond=0xa3f708) at pthread_cond_wait.c:508 }}
      {{#2 __pthread_cond_wait (cond=cond@entry=0xa3f708, mutex=mutex@entry=0xa3f6e0) at pthread_cond_wait.c:638 }}
      {{#3 0x00007f7fa4735ccb in suspend (ts=0xa3f6e0, p=0x95e4a0) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:393 }}
      {{#4 next_event_batch (p=0x95e4a0, can_block=true) at /home/kgiusti/work/proton/qpid-proton/c/src/proactor/epoll.c:2459 }}
      {{#5 0x00007f7fa47db54f in thread_run (arg=0x8d89f0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1106 }}
      {{#6 0x00007f7fa46b5432 in start_thread (arg=<optimized out>) at pthread_create.c:477 }}
      {{#7 0x00007f7fa41f96d3 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 }}

      {{Thread 1 (Thread 0x7f7fa3902300 (LWP 1257318)): }}
      {{#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 }}
      {{#1 0x00007f7fa411d895 in __GI_abort () at abort.c:79 }}
      #2 0x00007f7fa411d769 in __assert_fail_base (fmt=0x7f7fa428b0d8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x7f7fa47ee88b "result == 0", file=0x7f7fa47ee840 "/home/kgiusti/work/dispatch/qpid-dispat
      {{ch/src/posix/threading.c", line=58, function=<optimized out>) at assert.c:92 }}
      #3 0x00007f7fa412ce86 in _GI__assert_fail (assertion=assertion@entry=0x7f7fa47ee88b "result == 0", file=file@entry=0x7f7fa47ee840 "/home/kgiusti/work/dispatch/qpid-dispatch/src/posix/threading.c", line=line@
      {{entry=58, function=function@entry=0x7f7fa47ee998 <_PRETTY_FUNCTION_.10> "sys_mutex_lock") at assert.c:101 }}
      {{#4 0x00007f7fa47a79d1 in sys_mutex_lock (mutex=<optimized out>) at /home/kgiusti/work/dispatch/qpid-dispatch/src/posix/threading.c:58 }}
      {{#5 0x00007f7fa479eb1c in qd_message_stream_data_release (stream_data=0xcf0448) at /home/kgiusti/work/dispatch/qpid-dispatch/src/message.c:2640 }}
      {{#6 0x00007f7fa477c4a5 in qdr_http1_out_data_fifo_cleanup (out_data=out_data@entry=0xd96bb0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/adaptors/http1/http1_adaptor.c:152 }}
      {{#7 0x00007f7fa4782802 in _server_request_free (hreq=hreq@entry=0xd96b48) at /home/kgiusti/work/dispatch/qpid-dispatch/src/adaptors/http1/http1_server.c:1647 }}
      {{#8 0x00007f7fa47829b7 in _server_request_free (hreq=0xd96b48) at /home/kgiusti/work/dispatch/qpid-dispatch/src/adaptors/http1/http1_server.c:1645 }}
      {{#9 _process_request (hreq=0xd96b48) at /home/kgiusti/work/dispatch/qpid-dispatch/src/adaptors/http1/http1_server.c:734 }}
      {{#10 0x00007f7fa4782e43 in _handle_connection_events (e=<optimized out>, qd_server=<optimized out>, context=0xa4ba88) at /home/kgiusti/work/dispatch/qpid-dispatch/src/adaptors/http1/http1_server.c:659 }}
      {{#11 0x00007f7fa47d9311 in handle_event_with_context (context=<optimized out>, qd_server=<optimized out>, e=<optimized out>) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:802 }}
      {{#12 do_handle_raw_connection_event (qd_server=<optimized out>, e=<optimized out>) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:808 }}
      #13 handle (qd_server=qd_server@entry=0x8d89f0, e=e@entry=0xda4580, pn_conn=pn_conn@entry=0x0, ctx=ctx@entry=0x0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1089

      #14 0x00007f7fa47db668 in thread_run (arg=arg@entry=0x8d89f0) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1121
      #15 0x00007f7fa47db830 in qd_server_run (qd=<optimized out>) at /home/kgiusti/work/dispatch/qpid-dispatch/src/server.c:1501
      #16 0x00000000004026a8 in main_process (config_path=0x7ffd8647ba86 "TestBadEndpoints.conf", python_pkgdir=<optimized out>, test_hooks=<optimized out>, fd=2) at /home/kgiusti/work/dispatch/qpid-dispatch/router/s\
      rc/main.c:115
      #17 0x000000000040240b in main (argc=5, argv=0x7ffd8647a268) at /home/kgiusti/work/dispatch/qpid-dispatch/router/src/main.c:369

       

      Message content has apparently been freed:

       

      (gdb) p *stream_data->owning_message->content
      $9 = {
      lock = 0xef62c0,
      ref_count = 0,<<<<<<----------------------

       

       

        Attachments

        Issue Links

          Activity

            People

            • Assignee:
              kgiusti Ken Giusti
              Reporter:
              kgiusti Ken Giusti

              Dates

              • Created:
                Updated:
                Resolved:

                Issue deployment