Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-5794

Stress test crash: DCHECK in ProgressUpdater::Update

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Major
    • Resolution: Cannot Reproduce
    • Affects Version/s: Impala 2.9.0
    • Fix Version/s: None
    • Component/s: Backend
    • Labels:
    • Epic Color:
      ghx-label-5

      Description

      impalad hit a DCHECK failure during the stress test.

      F0808 22:23:04.227936 444839 progress-updater.cc:43] Check failed: delta >= 0 (-1 vs. 0)

      Stack trace:

      #0  0x00000033c7432625 in raise () from /lib64/libc.so.6
      No symbol table info available.
      #1  0x00000033c7433e05 in abort () from /lib64/libc.so.6
      No symbol table info available.
      #2  0x000000000297c854 in google::DumpStackTraceAndExit() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #3  0x00000000029732cd in google::LogMessage::Fail() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #4  0x0000000002974b72 in google::LogMessage::SendToLog() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #5  0x0000000002972ca7 in google::LogMessage::Flush() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #6  0x000000000297626e in google::LogMessageFatal::~LogMessageFatal() ()
              incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
      #7  0x0000000001679a6b in impala::ProgressUpdater::Update (this=0x7f133852b4a8, delta=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/progress-updater.cc:43
              _result = 0x7f01d282c080
              old_percentage = 6.9028712019688555e-310
              num_complete = 139616302725424
              new_percentage = 1785311752
      #8  0x0000000001b1613d in impala::Coordinator::BackendState::InstanceStats::Update (this=0x7efaf420f8e0, exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:479
              children = {<std::_Vector_base<impala::RuntimeProfile*, std::allocator<impala::RuntimeProfile*> >> = {_M_impl = {<std::allocator<impala::RuntimeProfile*>> = {<__gnu_cxx::new_allocator<impala::RuntimeProfile*>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f0e05e3e940, _M_finish = 0x7f0e05e3e968, 
                    _M_end_of_storage = 0x7f0e05e3e980}}, <No data fields>}
              thrift_exec_summary = @0x7f133852b4f8
              total = 0
              delta = -1
      #9  0x0000000001b13fcb in impala::Coordinator::BackendState::ApplyExecStatusReport (this=0x7ef1c88a2100, backend_exec_status=..., exec_summary=0x7f133852b4f8, scan_range_progress=0x7f133852b4a8, done=0x7f1216cb403f) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator-backend-state.cc:250
              instance_status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
              instance_idx = 4
              instance_stats = 0x7efaf420f8e0
              instance_exec_status = @0x7efae12bda00
              __for_range = @0x7f1216cb4498
              __for_begin = {_M_current = 0x7efae12bda00}
              __for_end = {_M_current = 0x7efae12bda80}
              l1 = {m = @0x7f133852b5c0}
              l2 = {m = @0x7ef1c88a21c0}
      #10 0x0000000001aff2db in impala::Coordinator::UpdateBackendExecStatus (this=0x7f133852b000, params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/coordinator.cc:951
              backend_state = 0x7ef1c88a2100
              done = 28
              failed_instance_id = {_vptr.TUniqueId = 0x7f1216cb4050, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = "\363\061\065\062\022S\332\353g\260\347\236Al\250\061", hi = 20621415, lo = 27705890}
              status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x5353350}, msg_ = 0x5352db0}, 
                static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f1216cb40af}
      #11 0x0000000001528b63 in impala::ImpalaServer::ReportExecStatus (this=0x142f5c00, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-server.cc:1147
              request_state = {<std::__shared_ptr<impala::ClientRequestState, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x7f0a0d9dcc00, _M_refcount = {_M_pi = 0x7efc59435800}}, <No data fields>}
      #12 0x0000000001599120 in impala::ImpalaInternalService::ReportExecStatus (this=0x14f38e60, return_val=..., params=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/impala-internal-service.cc:79
      No locals.
      #13 0x0000000001a6cee8 in impala::ImpalaInternalServiceProcessor::process_ReportExecStatus (this=0x12af0640, seqid=0, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1451
              ctx = 0x7f0a0909e7c0
              bytes = 0
              freer = {handler_ = 0x14f35110, context_ = 0x7f0a0909e7c0, method_ = 0x2f89ec0 "ImpalaInternalService.ReportExecStatus"}
              args = {_vptr.ImpalaInternalService_ReportExecStatus_args = 0x3fc0550, params = {_vptr.TReportExecStatusParams = 0x3fac7b0, static ascii_fingerprint = 0x2a19550 "479031394ED7C264DAC814BABAD737AD", static binary_fingerprint = "G\220\061\071N\327\302d\332\310\024\272\272\327\067\255", 
                  protocol_version = impala::ImpalaInternalServiceVersion::V1, query_id = {_vptr.TUniqueId = 0x3fa23b0, static ascii_fingerprint = 0x2a17fc0 "F33135321253DAEB67B0E79E416CA831", static binary_fingerprint = <same as static member of an already seen type>, hi = 2038521759236003358, lo = -6476124621472006144}, coord_state_idx = 3, 
                  instance_exec_status = {<std::_Vector_base<impala::TFragmentInstanceExecStatus, std::allocator<impala::TFragmentInstanceExecStatus> >> = {_M_impl = {<std::allocator<impala::TFragmentInstanceExecStatus>> = {<__gnu_cxx::new_allocator<impala::TFragmentInstanceExecStatus>> = {<No data fields>}, <No data fields>}, 
                        _M_start = 0x7efae12bda00, _M_finish = 0x7efae12bda80, _M_end_of_storage = 0x7efae12bda80}}, <No data fields>}, insert_exec_status = {_vptr.TInsertExecStatus = 0x3fac7d0, static ascii_fingerprint = 0x2a19410 "298B51CA995BEC745FCB22B330B4877B", static binary_fingerprint = <same as static member of an already seen type>, 
                    files_to_move = {_M_t = {
                        _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >> = {<No data fields>}, <No data fields>}, 
                          _M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb44c0, 
                            _M_right = 0x7f1216cb44c0}, _M_node_count = 0}}}, per_partition_status = {_M_t = {
                        _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, impala::TInsertPartitionStatus> > >> = {<No data fields>}, <No data fields>}, _M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, 
                            _M_left = 0x7f1216cb44f0, _M_right = 0x7f1216cb44f0}, _M_node_count = 0}}}, __isset = {per_partition_status = false}}, error_log = {_M_t = {
                      _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >> = {<No data fields>}, <No data fields>}, 
                        _M_key_compare = {<std::binary_function<impala::TErrorCode::type, impala::TErrorCode::type, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f1216cb4528, _M_right = 0x7f1216cb4528}, _M_node_count = 0}}}, __isset = {query_id = true, coord_state_idx = true, 
                    instance_exec_status = true, insert_exec_status = false, error_log = false}}, __isset = {params = true}}
              result = {_vptr.ImpalaInternalService_ReportExecStatus_result = 0x3fc0510, success = {_vptr.TReportExecStatusResult = 0x3fac790, static ascii_fingerprint = 0x2a191e8 "BA609E39B08A478BB74F0F877E3C5C4A", static binary_fingerprint = "\272`\236\071\260\212G\213\267O\017\207~<\\J", status = {_vptr.TStatus = 0x3fa3850, 
                    static ascii_fingerprint = 0x2a17a80 "A22BE3E84688C9DA4E00CC902B4EE818", static binary_fingerprint = <same as static member of an already seen type>, status_code = impala::TErrorCode::OK, 
                    error_msgs = {<std::_Vector_base<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >> = {
                        _M_impl = {<std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<__gnu_cxx::new_allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>}, 
                    __isset = {error_msgs = false}}, __isset = {status = false}}, __isset = {success = false}}
      #14 0x0000000001a6c4ce in impala::ImpalaInternalServiceProcessor::dispatchCall (this=0x12af0640, iprot=0x7f0cd3689a70, oprot=0x7eeb52785f80, fname=..., seqid=0, callContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1370
              pfn = {_M_node = 0x12af31c0}
      #15 0x0000000001200da2 in apache::thrift::TDispatchProcessor::process (this=0x12af0640, in=..., out=..., connectionContext=0x7eec0fc9dc80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/thrift-0.9.0-p9/include/thrift/TDispatchProcessor.h:121
              fname = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f01bde656c8 "ReportExecStatus"}}
              mtype = apache::thrift::protocol::T_CALL
              seqid = 0
      #16 0x00000000013cd0af in apache::thrift::server::TAcceptQueueServer::Task::run (this=0x7eea6ad998a0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/TAcceptQueueServer.cpp:76
              eventHandler = {px = 0x11d27980, pn = {pi_ = 0x142bfcc0}}
              connectionContext = 0x7eec0fc9dc80
      #17 0x00000000013c918b in impala::ThriftThread::RunRunnable (this=0x1637dc40, runnable=..., promise=0x7f1684dff570) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/rpc/thrift-thread.cc:64
      No locals.
      #18 0x00000000013ca8b7 in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0x7f0af77b66f0, p=0x1637dc40, a1=..., a2=0x7f1684dff570)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:280
      No locals.
      #19 0x00000000013ca74d in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0x7f0af77b6700, f=..., a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:392
      No locals.
      #20 0x00000000013ca499 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0x7f0af77b66f0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
              a = {<No data fields>}
      #21 0x00000000013ca3ac in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
              f = 0x7f0af77b66f0
      #22 0x00000000013d7b2c in boost::function0<void>::operator() (this=0x7f1216cb4c20) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
      No locals.
      #23 0x00000000016ad925 in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f1684dff360) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:330
              system_tid = 444839
              category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x14d61758 "thrift-server"}}
              thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x53b7c00, _M_refcount = {_M_pi = 0x53adc80}}, <No data fields>}
              name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f0cd3688858 "backend-5064"}}
      #24 0x00000000016b642c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f04da0befc0, f=@0x7f04da0befb8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
      No locals.
      #25 0x00000000016b636f in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f04da0befb8)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
      ---Type <return> to continue, or q <return> to quit---
              a = {<No data fields>}
      #26 0x00000000016b6332 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f04da0bee00)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
      No locals.
      #27 0x0000000001ba3dca in thread_proxy ()
      No symbol table info available.
      #28 0x00000033c7807aa1 in start_thread () from /lib64/libpthread.so.0
      No symbol table info available.
      #29 0x00000033c74e893d in clone () from /lib64/libc.so.6
      No symbol table info available.

        Attachments

          Activity

            People

            • Assignee:
              Unassigned
              Reporter:
              mmulder Matthew Mulder
            • Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: