Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-1019

Failed DCHECK in disk-io-mgr-reader-context.cc:174] num_used_buffers_ < 0: #used=-1 during cancellation HDFS cached data

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Blocker
    • Resolution: Fixed
    • Affects Version/s: Impala 1.4
    • Fix Version/s: Impala 1.4
    • Component/s: None
    • Labels:
      None

      Description

      This does not reproduce 100% of the time, but I found a query that hits this fairly frequently (have not had a chance to find a minimal repro):

      with t1 as (select int_col x, bigint_col y from functional.alltypes limit 2), t2 as (select int_col x, bigint_col y from functional.alltypestiny limit 2), t3 as (select int_col x, bigint_col y from functional.alltypessmall limit 2) select * from t1, t2, t3 where t1.x = t2.x and t2.x = t3.x;
      

      Where "alltypestiny" is fully cached.

      W0524 04:37:20.026154 31164 disk-io-mgr-reader-context.cc:174] num_used_buffers_ < 0: #used=-1
      F0524 04:37:20.026307 31164 disk-io-mgr.cc:476] Check failed: reader->Validate()
      
        Reader: 0x13a63860 (state=Active status_=OK #ready_buffers=0 #used_buffers=-1 #num_buffers_in_reader=1 #finished_scan_ranges=0 #disk_with_ranges=0 #disks=0
         0: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         1: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         2: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         3: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         4: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         5: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0)
      I0524 04:37:20.026330 17955 impala-server.cc:1006] ReportExecStatus(): Received report for unknown query ID (probably closed or cancelled). (query_id: da421f3093131e12:da5a63d7384a8ca0, backend: 0, instance: da421f3093131e12:da5a63d7384a8ca1 done: true)
      W0524 04:37:20.032737 31141 disk-io-mgr-reader-context.cc:174] num_used_buffers_ < 0: #used=-1
      F0524 04:37:20.032845 31141 disk-io-mgr-reader-context.cc:26] Check failed: Validate()
      
        Reader: 0x1510fe00 (state=Active status_=OK #ready_buffers=1 #used_buffers=-1 #num_buffers_in_reader=1 #finished_scan_ranges=0 #disk_with_ranges=0 #disks=0
         0: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         1: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         2: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         3: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         4: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0
         5: is_on_queue=0 done=1 #num_remaining_scan_ranges=0 #in_flight_ranges=0 #unstarted_ranges=0 #reading_threads=0)
      
          __in_chrg=<value optimized out>) at src/logging.cc:1817
      #7  0x0000000000e6254b in impala::DiskIoMgr::GetNextRange (this=0x71563c0, reader=0x13a63860, 
          range=0x7f01b9016808)
          at /data/2/jenkins/workspace/impala-master-64bit-gerrit-merge-submit/BUILD_TYPE/DEBUG/label/impala/repos/Impala/be/src/runtime/disk-io-mgr.cc:476
      #8  0x00000000012b6c54 in impala::HdfsScanNode::ScannerThread (this=0x282ed180)
          at /data/2/jenkins/workspace/impala-master-64bit-gerrit-merge-submit/BUILD_TYPE/DEBUG/label/impala/repos/Impala/be/src/exec/hdfs-scan-node.cc:803
      #9  0x00000000012cae29 in boost::_mfi::mf0<void, impala::HdfsScanNode>::operator() (this=0x7f01b9016cd8, 
          p=0x282ed180) at /usr/include/boost/bind/mem_fn_template.hpp:49
      #10 0x00000000012ca544 in boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> >::operator()<boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list0> (this=0x7f01b9016ce8, f=..., a=...)
          at /usr/include/boost/bind/bind.hpp:253
      #11 0x00000000012c93a7 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> > >::operator() (this=0x7f01b9016cd8)
          at /usr/include/boost/bind/bind_template.hpp:20
      #12 0x00000000012c7dea in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::HdfsScanNode>, boost::_bi::list1<boost::_bi::value<impala::HdfsScanNode*> > >, void>::invoke
          (function_obj_ptr=...) at /usr/include/boost/function/function_template.hpp:153
      #13 0x0000000000e28f0b in boost::function0<void>::operator() (this=0x7f01b9016cd0)
          at /usr/include/boost/function/function_template.hpp:1013
      #14 0x000000000101d6ec in impala::Thread::SuperviseThread (name="scanner-thread(2)", category="hdfs-scan-node", 
          functor=..., thread_started=0x7f01b0d08800)
          at /data/2/jenkins/workspace/impala-master-64bit-gerrit-merge-submit/BUILD_TYPE/DEBUG/label/impala/repos/Impala/be/src/util/thread.cc:263
      #15 0x00000000010256e0 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::string &, const std::string &, impala::Thread::ThreadFunctor, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x5e6d630, f=@0x5e6d628, a=...)
          at /usr/include/boost/bind/bind.hpp:457
      #16 0x0000000001025627 in boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x5e6d628) at /usr/include/boost/bind/bind_template.hpp:20
      #17 0x00000000010255b4 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x5e6d4a0) at /usr/include/boost/thread/detail/thread.hpp:61
      #18 0x00000000013b6264 in thread_proxy ()
      #19 0x0000003f52207851 in start_thread () from /lib64/libpthread.so.0
      #20 0x0000003f51ae894d in clone () from /lib64/libc.so.6
      

        Attachments

          Activity

            People

            • Assignee:
              nong_impala_60e1 Nong Li
              Reporter:
              lskuff Lenni Kuff
            • Votes:
              0 Vote for this issue
              Watchers:
              0 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: