Details
-
Bug
-
Status: Resolved
-
Critical
-
Resolution: Works for Me
-
Impala 2.2
-
None
-
None
Description
This happened while running tpch q3. This is the first time this happened (the query runs fine sometimes).
#0 0x00007ff4287fce37 in __GI_raise (sig=sig@entry=6)
at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
56 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ff4287fce37 in __GI_raise (sig=sig@entry=6)
at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007ff4287fe528 in __GI_abort () at abort.c:89
#2 0x00007ff4279decf5 in os::abort(bool) ()
from /usr/lib/jvm/java-7-oracle/jre/lib/amd64/server/libjvm.so
#3 0x00007ff427b604f7 in VMError::report_and_die() ()
from /usr/lib/jvm/java-7-oracle/jre/lib/amd64/server/libjvm.so
#4 0x00007ff4279e3c0f in JVM_handle_linux_signal ()
from /usr/lib/jvm/java-7-oracle/jre/lib/amd64/server/libjvm.so
#5 <signal handler called>
#6 tcmalloc::CentralFreeList::FetchFromSpans (
this=this@entry=0x2d25500 <tcmalloc::Static::central_cache_+7296>)
at src/central_freelist.cc:298
#7 0x00000000020b1997 in tcmalloc::CentralFreeList::RemoveRange (
this=0x2d25500 <tcmalloc::Static::central_cache_+7296>,
start=start@entry=0x7ff3a9cb0688, end=end@entry=0x7ff3a9cb0690, N=32)
at src/central_freelist.cc:269
#8 0x00000000020acc22 in tcmalloc::ThreadCache::FetchFromCentralCache (
this=this@entry=0xb619880, cl=6, byte_size=80) at src/thread_cache.cc:162
#9 0x00000000020cb098 in Allocate (cl=<optimized out>, size=<optimized out>,
this=0xb619880) at src/thread_cache.h:341
#10 do_malloc (size=<optimized out>) at src/tcmalloc.cc:1068
#11 cpp_alloc (nothrow=false, size=80) at src/tcmalloc.cc:1354
#12 tc_new (size=80) at src/tcmalloc.cc:1530
#13 0x00000000016ddfbc in llvm::BitcodeReader::ParseFunctionBody(llvm::Function*) ()
#14 0x00000000016e50e3 in llvm::BitcodeReader::Materialize(llvm::GlobalValue*, std::string*) ()
#15 0x00000000016e5225 in llvm::BitcodeReader::MaterializeModule(llvm::Module*, std::string*) ()
#16 0x0000000001f71089 in llvm::Module::MaterializeAllPermanently(std::string*) ()
#17 0x00000000016e4b59 in llvm::ParseBitcodeFile(llvm::MemoryBuffer*, llvm::LLVMContext&, std::string*) ()
#18 0x0000000000ee88b5 in impala::LlvmCodeGen::LoadModule (codegen=0x7961700,
file="/home/casey/Impala/llvm-ir/impala-sse.ll", module=0x7ff3a9cb11e8)
at /home/casey/Impala/be/src/codegen/llvm-codegen.cc:150
#19 0x0000000000ee851e in impala::LlvmCodeGen::LoadFromFile (pool=0xb462dc0,
file="/home/casey/Impala/llvm-ir/impala-sse.ll",
id="9540fd111faf15ad:74ea16fba0e293b9", codegen=0xc569090)
at /home/casey/Impala/be/src/codegen/llvm-codegen.cc:126
#20 0x0000000000ee8f70 in impala::LlvmCodeGen::LoadImpalaIR (pool=0xb462dc0,
id="9540fd111faf15ad:74ea16fba0e293b9", codegen_ret=0xc569090)
at /home/casey/Impala/be/src/codegen/llvm-codegen.cc:189
#21 0x0000000000fc4913 in impala::RuntimeState::CreateCodegen (this=0xc568e00)
at /home/casey/Impala/be/src/runtime/runtime-state.cc:193
#22 0x0000000000fc5f2d in impala::RuntimeState::GetCodegen (this=0xc568e00,
codegen=0x7ff3a9cb17d0, initialize=true)
at /home/casey/Impala/be/src/runtime/runtime-state.cc:315
#23 0x00000000015ebbf4 in impala::PartitionedAggregationNode::Prepare (this=0xb08c600,
state=0xc568e00) at /home/casey/Impala/be/src/exec/partitioned-aggregation-node.cc:102
#24 0x00000000015347d5 in impala::ExecNode::Prepare (this=0x9ec3440, state=0xc568e00)
at /home/casey/Impala/be/src/exec/exec-node.cc:149
#25 0x000000000162383b in impala::TopNNode::Prepare (this=0x9ec3440, state=0xc568e00)
at /home/casey/Impala/be/src/exec/topn-node.cc:56
#26 0x00000000014fb54a in impala::PlanFragmentExecutor::Prepare (this=0x9a08210,
request=...) at /home/casey/Impala/be/src/runtime/plan-fragment-executor.cc:233
#27 0x0000000001135740 in impala::FragmentMgr::FragmentExecState::Prepare (this=0x9a08000,
exec_params=...) at /home/casey/Impala/be/src/service/fragment-exec-state.cc:43
#28 0x000000000112c78e in impala::FragmentMgr::ExecPlanFragment (this=0x94b66c0,
exec_params=...) at /home/casey/Impala/be/src/service/fragment-mgr.cc:51
The query completed two times but crashed on the third:
casey@casey-desktop:~/Impala/tests/query_test$ py.test test_mem_usage_scaling.py -k q3=================================== test session starts ====================================
platform linux2 -- Python 2.7.8 -- py-1.4.24 -- pytest-2.6.2 -- /usr/bin/python
plugins: xdist
collected 124 items
test_mem_usage_scaling.py::TestTpchMemLimitError::test_low_mem_limit_q3[mem_limit: 100 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none] PASSED
test_mem_usage_scaling.py::TestTpchMemLimitError::test_low_mem_limit_q3[mem_limit: 150 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none] PASSED
test_mem_usage_scaling.py::TestTpchMemLimitError::test_low_mem_limit_q3[mem_limit: 180 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none] FAILEDTraceback (most recent call last):
File "/usr/lib/python2.7/logging/__init__.py", line 880, in emit
stream.write(fs % msg)
File "/usr/local/lib/python2.7/dist-packages/_pytest/capture.py", line 234, in write
self.buffer.write(obj)
ValueError: I/O operation on closed file
Logged from file impala_connection.py, line 150
========================================= FAILURES =========================================
TestTpchMemLimitError.test_low_mem_limit_q3[mem_limit: 180 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
test_mem_usage_scaling.py:140: in test_low_mem_limit_q3
self.low_memory_limit_test(vector, 'tpch-q3', self.MIN_MEM_FOR_TPCH['Q3']);
test_mem_usage_scaling.py:131: in low_memory_limit_test
assert TestTpchMemLimitError.EXPECTED_ERROR_MSG in str(e)
E assert 'Memory limit exceeded' in "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes"
E + where 'Memory limit exceeded' = TestTpchMemLimitError.EXPECTED_ERROR_MSG
E + and "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes" = str(ImpalaBeeswaxException())
----------------------------------- Captured stderr call -----------------------------------
-- executing against localhost:21000
use tpch_parquet;
SET batch_size=0;
SET num_nodes=0;
SET disable_codegen=False;
SET abort_on_error=1;
SET mem_limit=180m;
SET exec_single_node_rows_threshold=0;
MainThread: Query Name:
TPCH-Q3
-- executing against localhost:21000
select
l_orderkey,
sum(l_extendedprice * (1 - l_discount)) as revenue,
o_orderdate,
o_shippriority
from
customer,
orders,
lineitem
where
c_mktsegment = 'BUILDING'
and c_custkey = o_custkey
and l_orderkey = o_orderkey
and o_orderdate < '1995-03-15'
and l_shipdate > '1995-03-15'
group by
l_orderkey,
o_orderdate,
o_shippriority
order by
revenue desc,
o_orderdate
limit 10;
================================= short test summary info ==================================
FAIL test_mem_usage_scaling.py::TestTpchMemLimitError::()::test_low_mem_limit_q3[mem_limit: 180 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
!!!!!!!!!!!!!!!!!!!!!!!!!! Interrupted: stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!
============================== 114 tests deselected by '-kq3' ==============================
============= 1 failed, 2 passed, 114 deselected, 3 warnings in 288.67 seconds =============