Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-2818

Query cancellation DCHECK while clearing object pool: expr-context.cc:41] Check failed: !prepared_ || closed_

    Details

      Description

      The impala-cdh5-integration build (http://sandbox.jenkins.cloudera.com/job/impala-cdh5-integration/319) crash. It looks like query cancellation triggered a crash. The stack trace is:

      #0  0x0000003f51a328e5 in raise () from /lib64/libc.so.6
      #1  0x0000003f51a340c5 in abort () from /lib64/libc.so.6
      #2  0x00000000021826c9 in google::DumpStackTraceAndExit() ()
      #3  0x000000000217aecd in google::LogMessage::Fail() ()
      #4  0x000000000217d7f6 in google::LogMessage::SendToLog() ()
      #5  0x000000000217a9ed in google::LogMessage::Flush() ()
      #6  0x000000000217e29e in google::LogMessageFatal::~LogMessageFatal() ()
      #7  0x000000000100a627 in impala::ExprContext::~ExprContext (this=0x1869c180, __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/exprs/expr-context.cc:41
      #8  0x000000000100987a in impala::ObjectPool::SpecificElement<impala::ExprContext>::~SpecificElement (this=0xc8c4940,
          __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/common/object-pool.h:65
      #9  0x00000000010098de in impala::ObjectPool::SpecificElement<impala::ExprContext>::~SpecificElement (this=0xc8c4940,
          __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/common/object-pool.h:66
      #10 0x0000000001273c81 in impala::ObjectPool::Clear (this=0x27d24840)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/common/object-pool.h:51
      #11 0x0000000001273bf9 in impala::ObjectPool::~ObjectPool (this=0x27d24840, __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/common/object-pool.h:34
      #12 0x000000000127bd57 in boost::checked_delete<impala::ObjectPool> (x=0x27d24840)
          at /tmp/impala-deps/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #13 0x0000000001279bbf in boost::scoped_ptr<impala::ObjectPool>::~scoped_ptr (this=0xa7c0d08,
          __in_chrg=<value optimized out>) at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #14 0x00000000012a513c in impala::RuntimeState::~RuntimeState (this=0xa7c0d00, __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/runtime/runtime-state.cc:94
      #15 0x00000000013f71a5 in boost::checked_delete<impala::RuntimeState> (x=0xa7c0d00)
          at /tmp/impala-deps/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #16 0x00000000013f470d in boost::scoped_ptr<impala::RuntimeState>::~scoped_ptr (this=0xe78f428,
          __in_chrg=<value optimized out>) at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #17 0x00000000015f9a0e in impala::PlanFragmentExecutor::~PlanFragmentExecutor (this=0xe78f2c0,
          __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/runtime/plan-fragment-executor.cc:71
      #18 0x00000000015d8f6b in boost::checked_delete<impala::PlanFragmentExecutor> (x=0xe78f2c0)
          at /tmp/impala-deps/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #19 0x00000000015d56d5 in boost::scoped_ptr<impala::PlanFragmentExecutor>::~scoped_ptr (this=0x36bb3470,
          __in_chrg=<value optimized out>) at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #20 0x00000000015c52bb in impala::Coordinator::~Coordinator (this=0x36bb3100, __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/runtime/coordinator.cc:254
      #21 0x00000000013f711e in boost::checked_delete<impala::Coordinator> (x=0x36bb3100)
          at /tmp/impala-deps/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #22 0x00000000013f4665 in boost::scoped_ptr<impala::Coordinator>::~scoped_ptr (this=0x8cec02a8,
          __in_chrg=<value optimized out>) at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #23 0x00000000013e4db0 in impala::ImpalaServer::QueryExecState::~QueryExecState (this=0x8cec0000,
          __in_chrg=<value optimized out>)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/query-exec-state.cc:111
      #24 0x00000000013a99c2 in boost::checked_delete<impala::ImpalaServer::QueryExecState> (x=0x8cec0000)
          at /tmp/impala-deps/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #25 0x00000000013b0158 in boost::detail::sp_counted_impl_p<impala::ImpalaServer::QueryExecState>::dispose (this=0x8b4772e0)
          at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/detail/sp_counted_impl.hpp:78
      #26 0x0000000000fa0f98 in boost::detail::sp_counted_base::release (this=0x8b4772e0)
          at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp:146
      #27 0x0000000000fa1027 in boost::detail::shared_count::~shared_count (this=0x7f5c9c343e68, __in_chrg=<value optimized out>)
          at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/detail/shared_count.hpp:443
      #28 0x0000000001388f6e in boost::shared_ptr<impala::ImpalaServer::QueryExecState>::~shared_ptr (this=0x7f5c9c343e60,
          __in_chrg=<value optimized out>) at /tmp/impala-deps/boost-1.57.0/include/boost/smart_ptr/shared_ptr.hpp:323
      #29 0x00000000013c99fa in impala::ImpalaServer::CloseOperation (this=0x88c2000, return_val=..., request=...)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/impala-hs2-server.cc:972
      #30 0x00000000014040e3 in impala::ChildQuery::Cancel (this=0x4e888380)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/child-query.cc:176
      #31 0x00000000013ebaac in impala::ImpalaServer::QueryExecState::Cancel (this=0xd9f8000, cause=0x0)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/query-exec-state.cc:819
      #32 0x0000000001373bee in impala::ImpalaServer::CancelInternal (this=0x88c2000, query_id=..., check_inflight=true, cause=0x0)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/impala-server.cc:966
      #33 0x00000000013dda1a in impala::ImpalaServer::Cancel (this=0x88c2000, tstatus=..., query_handle=...)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/service/impala-beeswax-server.cc:443
      #34 0x000000000152c71c in impala::ImpalaServiceProcessor::process_Cancel (this=0x8fd37c0, seqid=0, iprot=0x12aed200,
          oprot=0x12aec280, callContext=0xc83ad00)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/generated-sources/gen-cpp/ImpalaService.cpp:1702
      #35 0x000000000152c4fd in impala::ImpalaServiceProcessor::dispatchCall (this=0x8fd37c0, iprot=0x12aed200, oprot=0x12aec280,
          fname=..., seqid=0, callContext=0xc83ad00)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/generated-sources/gen-cpp/ImpalaService.cpp:1675
      #36 0x000000000137ea14 in apache::thrift::TDispatchProcessor::process (this=0x8fd37c0, in=..., out=...,
          connectionContext=0xc83ad00) at /tmp/impala-deps/thrift-0.9.0-p2/include/thrift/TDispatchProcessor.h:121
      #37 0x000000000211f07b in apache::thrift::server::TThreadPoolServer::Task::run() ()
      #38 0x0000000002106f49 in apache::thrift::concurrency::ThreadManager::Worker::run() ()
      #39 0x000000000125bd39 in impala::ThriftThread::RunRunnable (this=0x9a43200, runnable=..., promise=0x7fff79f2a8b0)
      ---Type <return> to continue, or q <return> to quit---
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/rpc/thrift-thread.cc:61
      #40 0x000000000125d541 in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0xa537bf0, p=0x9a43200, a1=..., a2=0x7fff79f2a8b0)
          at /tmp/impala-deps/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:280
      #41 0x000000000125d3d7 in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0xa537c00, f=..., a=...) at /tmp/impala-deps/boost-1.57.0/include/boost/bind/bind.hpp:392
      #42 0x000000000125d123 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0xa537bf0) at /tmp/impala-deps/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #43 0x000000000125d036 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...)
          at /tmp/impala-deps/boost-1.57.0/include/boost/function/function_template.hpp:153
      #44 0x0000000001292e30 in boost::function0<void>::operator() (this=0x7f5c9c344cc0)
          at /tmp/impala-deps/boost-1.57.0/include/boost/function/function_template.hpp:767
      #45 0x00000000014b7be3 in impala::Thread::SuperviseThread (name=..., category=..., functor=...,
          thread_started=0x7fff79f2a6a0)
          at /data/3/jenkins/workspace/impala-cdh5-integration/repos/Impala/be/src/util/thread.cc:316
      #46 0x00000000014c11c6 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (
          this=0x9d35bc0, f=@0x9d35bb8, a=...) at /tmp/impala-deps/boost-1.57.0/include/boost/bind/bind.hpp:457
      #47 0x00000000014c1109 in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x9d35bb8)
          at /tmp/impala-deps/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #48 0x00000000014c10cc in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x9d35a00)
          at /tmp/impala-deps/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
      

      The failed tests were:

          failure.test_failpoints.TestFailpoints.test_failpoints[target_node: ('SCAN HDFS', [4, 2]) | table_format: avro/snap/block | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | query_type: SELECT | location: OPEN | action: CANCEL]
          query_test.test_cancellation.TestCancellationParallel.test_cancel_select[table_format: avro/snap/block | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | query_type: SELECT | cancel_delay: 2 | action: None | query: compute stats lineitem]
          metadata.test_col_stats.TestColStats.test_incompatible_col_stats[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]
          query_test.test_chars.TestCharFormats.test_char_format[exec_option: {'disable_codegen': True, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]
          failure.test_failpoints.TestFailpoints.test_failpoints[target_node: ('AGGREGATE', [6]) | table_format: parquet/none | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | query_type: SELECT | location: PREPARE | action: FAIL]
          query_test.test_aggregation.TestAggregation.test_aggregation[data_type: tinyint | agg_func: min | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
          metadata.test_col_stats.TestColStats.test_incompatible_col_stats[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]
          query_test.test_decimal_queries.TestDecimalQueries.test_queries[batch_size: 0 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]
          query_test.test_chars.TestCharFormats.test_char_format[exec_option: {'disable_codegen': True, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
          verifiers.test_verify_metrics.TestValidateMetrics.test_metrics_are_zero
          verifiers.test_verify_metrics.TestValidateMetrics.test_num_unused_buffers
      

      I have the core file if needed.

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                dhecht Dan Hecht
                Reporter:
                tarmstrong Tim Armstrong
              • Votes:
                0 Vote for this issue
                Watchers:
                6 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: