Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-7473

RawValue::PrintValue() hits general protection fault with VLOG level 3

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Critical
    • Resolution: Fixed
    • Affects Version/s: Impala 2.13.0
    • Fix Version/s: Impala 3.2.0
    • Component/s: Backend
    • Labels:

      Description

      DecimalValue does not seem to be 16 bytes aligned and impala hits a GPF. I could only recreate this issue only with logging level set to 3.
      Here is a recreation of this on Impala version - 2.13.0

      [anuj-3.gce.cloudera.com:21000] > select * from tem_cja.fact_starts_channel limit 2;
       Query: select * from tem_cja.fact_starts_channel limit 2
       Query submitted at: 2018-08-21 10:33:52 (Coordinator: [http://anuj-3.gce.cloudera.com:25000|http://anuj-3.gce.cloudera.com:25000/])
       Query progress can be monitored at: [http://anuj-3.gce.cloudera.com:25000/query_plan?query_id=1b46ea60e56fb01c:b60b7bfa00000000]
       Error communicating with impalad: TSocket read 0 bytes
       [Not connected] >
      
      (gdb) bt
       #0 0x00007f59a9645277 in raise () from /lib64/libc.so.6
       #1 0x00007f59a9646968 in abort () from /lib64/libc.so.6
       #2 0x00007f59ac335f35 in os::abort(bool) () from /usr/java/jdk1.8.0_162-cloudera/jre/lib/amd64/server/libjvm.so
       #3 0x00007f59ac4d9f33 in VMError::report_and_die() () from /usr/java/jdk1.8.0_162-cloudera/jre/lib/amd64/server/libjvm.so
       #4 0x00007f59ac33c22f in JVM_handle_linux_signal () from /usr/java/jdk1.8.0_162-cloudera/jre/lib/amd64/server/libjvm.so
       #5 0x00007f59ac332253 in signalHandler(int, siginfo*, void*) () from /usr/java/jdk1.8.0_162-cloudera/jre/lib/amd64/server/libjvm.so
       #6 <signal handler called>
       #7 0x0000000000b67f6b in impala::RawValue::PrintValue(void const*, impala::ColumnType const&, int, std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >*) ()
       #8 0x0000000000b686ec in impala::RawValue::PrintValue(void const*, impala::ColumnType const&, int, std::string*) ()
       #9 0x0000000000cd1415 in impala::PrintTuple(impala::Tuple const*, impala::TupleDescriptor const&) ()
       #10 0x0000000000cd16cb in impala::PrintRow(impala::TupleRow*, impala::RowDescriptor const&) ()
       #11 0x0000000000b69718 in impala::RowBatch::VLogRows(std::string const&) ()
       #12 0x0000000000badcdd in impala::FragmentInstanceState::ExecInternal() ()
       #13 0x0000000000bb07bf in impala::FragmentInstanceState::Exec() ()
       #14 0x0000000000b9e85a in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) ()
       #15 0x0000000000d60f1f in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*) ()
       #16 0x0000000000d6171a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long>*> > > >::run() ()
       #17 0x00000000012d930a in thread_proxy ()
       #18 0x00007f59a99e3e25 in start_thread () from /lib64/libpthread.so.0
       #19 0x00007f59a970dbad in clone () from /lib64/libc.so.6
       (gdb) quit
      

      Here is a better stack trace from a CTAS query that I ran on Impala 2.10.0 -

      Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
       Core was generated by `/home/anuj/Impala/be/build/latest/service/impalad -kudu_master_hosts 127.0.0.1'.
       Program terminated with signal SIGABRT, Aborted.
       #0 0x00007f0597161c37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
       56 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.
       (gdb) bt
       #0 0x00007f0597161c37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
       #1 0x00007f0597165028 in __GI_abort () at abort.c:89
       #2 0x00007f05996f63c5 in os::abort(bool) () from /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so
       #3 0x00007f059989a5b3 in VMError::report_and_die() () from /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so
       #4 0x00007f05996fc70f in JVM_handle_linux_signal () from /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so
       #5 0x00007f05996f2653 in signalHandler(int, siginfo*, void*) () from /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so
       #6 <signal handler called>
       #7 ToString (scale=5, precision=<optimized out>, this=0x1075e264) at /home/anuj/Impala/be/src/runtime/decimal-value.inline.h:468
       #8 ToString (type=..., type=..., this=0x1075e264) at /home/anuj/Impala/be/src/runtime/decimal-value.inline.h:452
       #9 impala::RawValue::PrintValue (value=value@entry=0x1075e264, type=..., scale=scale@entry=-1, stream=stream@entry=0x7f04f5abb390)
       at /home/anuj/Impala/be/src/runtime/raw-value.cc:300
       #10 0x0000000000a00ac6 in impala::RawValue::PrintValue (value=0x1075e264, type=..., scale=scale@entry=-1, str=str@entry=0x7f04f5abb590)
       at /home/anuj/Impala/be/src/runtime/raw-value.cc:108
       #11 0x0000000000b594a2 in impala::PrintTuple (t=0x1075e174, d=...) at /home/anuj/Impala/be/src/util/debug-util.cc:198
       #12 0x0000000000b59714 in impala::PrintRow (row=0xd930008, d=...) at /home/anuj/Impala/be/src/util/debug-util.cc:211
       #13 0x0000000000a38edc in impala::RowBatch::VLogRows (this=0xc44eee8, context=<error reading variable: Cannot access memory at address 0x30>)
       at /home/anuj/Impala/be/src/runtime/row-batch.cc:446
       #14 0x0000000000a51adc in impala::FragmentInstanceState::ExecInternal (this=this@entry=0x75f4d00) at /home/anuj/Impala/be/src/runtime/fragment-instance-state.cc:273
       #15 0x0000000000a54f19 in impala::FragmentInstanceState::Exec (this=this@entry=0x75f4d00) at /home/anuj/Impala/be/src/runtime/fragment-instance-state.cc:89
       #16 0x0000000000a316d8 in impala::QueryState::ExecFInstance (this=0x7082d00, fis=0x75f4d00) at /home/anuj/Impala/be/src/runtime/query-state.cc:369
       #17 0x0000000000bd6fa2 in operator() (this=0x7f04f5abbda0) at /home/anuj/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
       #18 impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) (name=..., category=..., functor=..., 
       thread_started=0x7f04f62bc210) at /home/anuj/Impala/be/src/util/thread.cc:352
       #19 0x0000000000bd7704 in operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>
       (f=@0x82457b8: 0xbd6de0 <impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*)>, a=<synthetic pointer>, 
       this=0x82457c0) at /home/anuj/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
       #20 operator() (this=0x82457b8) at /home/anuj/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
       #21 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x8245600)
       at /home/anuj/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
       #22 0x0000000000e6414a in thread_proxy ()
       #23 0x00007f05974fc184 in start_thread (arg=0x7f04f5abc700) at pthread_create.c:312
       #24 0x00007f059722903d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
       (gdb) f 8
       #8 ToString (type=..., type=..., this=0x1075e264) at /home/anuj/Impala/be/src/runtime/decimal-value.inline.h:452
       452 return ToString(type.precision, type.scale);
       (gdb) f 7
       #7 ToString (scale=5, precision=<optimized out>, this=0x1075e264) at /home/anuj/Impala/be/src/runtime/decimal-value.inline.h:468
       468 T remaining_value = value_;
       (gdb) f 6
       #6 <signal handler called>
       (gdb) f 7
       #7 ToString (scale=5, precision=<optimized out>, this=0x1075e264) at /home/anuj/Impala/be/src/runtime/decimal-value.inline.h:468
       468 T remaining_value = value_;
       (gdb) p value_
       $1 = 0x000000000000000000df88b350ffb87f
       

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                tarmstrong Tim Armstrong
                Reporter:
                anujphadke Anuj Phadke
              • Votes:
                0 Vote for this issue
                Watchers:
                5 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: