Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Duplicate
-
Impala 2.8.0
Description
We've seen two crashes in Jenkins builds with similar symptoms:
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/main.py", line 84, in wrap_session
08:57:44 INTERNALERROR> doit(config, session)
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/main.py", line 122, in _main
08:57:44 INTERNALERROR> config.hook.pytest_runtestloop(session=session)
08:57:44 Traceback (most recent call last):
08:57:44 File "/usr/lib64/python2.6/logging/__init__.py", line 797, in emit
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 521, in __call__
08:57:44 INTERNALERROR> return self._docall(self.methods, kwargs)
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 528, in _docall
08:57:44 INTERNALERROR> firstresult=self.firstresult).execute()
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 394, in execute
08:57:44 INTERNALERROR> res = method(*args)
08:57:44 INTERNALERROR> File "<remote exec>", line 47, in pytest_runtestloop
08:57:44 INTERNALERROR> File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/execnet/gateway_base.py", line 737, in receive
08:57:44 INTERNALERROR> raise self._getremoteerror() or EOFError()
08:57:44 INTERNALERROR> EOFError
08:57:44 stream.write(fs % msg)
08:57:44 File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/capture.py", line 229, in write
08:57:44 self.buffer.write(obj)
08:57:44 ValueError: I/O operation on closed file
08:57:44
08:57:44
08:57:44 ==================================== ERRORS ====================================
08:57:44 ERROR at teardown of TestRefreshPartition.test_drop_hive_partition_and_refresh[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]
08:57:44 [gw1] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
08:57:44 conftest.py:270: in cleanup
08:57:44 request.instance.execute_query_expect_success(request.instance.client, "use default")
08:57:44 common/impala_test_suite.py:433: in wrapper
08:57:44 return function(*args, **kwargs)
08:57:44 common/impala_test_suite.py:440: in execute_query_expect_success
08:57:44 result = cls.__execute_query(impalad_client, query, query_options)
08:57:44 common/impala_test_suite.py:525: in __execute_query
08:57:44 return impalad_client.execute(query, user=user)
08:57:44 common/impala_connection.py:160: in execute
08:57:44 return self.__beeswax_client.execute(sql_stmt, user=user)
08:57:44 beeswax/impala_beeswax.py:173: in execute
08:57:44 handle = self.__execute_query(query_string.strip(), user=user)
08:57:44 beeswax/impala_beeswax.py:337: in __execute_query
08:57:44 handle = self.execute_query_async(query_string, user=user)
08:57:44 beeswax/impala_beeswax.py:333: in execute_query_async
08:57:44 return self.__do_rpc(lambda: self.imp_service.query(query,))
08:57:44 beeswax/impala_beeswax.py:465: in __do_rpc
08:57:44 raise ImpalaBeeswaxException(self.__build_error_message(u), u)
08:57:44 E ImpalaBeeswaxException: ImpalaBeeswaxException:
08:57:44 E INNER EXCEPTION: <class 'socket.error'>
08:57:44 E MESSAGE: [Errno 104] Connection reset by peer
08:57:44 ---------------------------- Captured stderr setup -----------------------------
08:57:44 SET sync_ddl=False;
08:57:44 -- executing against localhost:21000
08:57:44 DROP DATABASE IF EXISTS `test_drop_hive_partition_and_refresh_e85660d6` CASCADE;
08:57:44
08:57:44 SET sync_ddl=False;
08:57:44 -- executing against localhost:21000
08:57:44 CREATE DATABASE `test_drop_hive_partition_and_refresh_e85660d6`;
08:57:44
08:57:44 MainThread: Created database "test_drop_hive_partition_and_refresh_e85660d6" for test ID "metadata/test_refresh_partition.py::TestRefreshPartition::()::test_drop_hive_partition_and_refresh[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]"
08:57:44 ----------------------------- Captured stderr call -----------------------------
08:57:44 -- executing against localhost:21000
08:57:44 create table test_drop_hive_partition_and_refresh_e85660d6.partition_test_table (x int) partitioned by (y int, z int);
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 alter table test_drop_hive_partition_and_refresh_e85660d6.partition_test_table add partition (y=333, z=5309);
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 refresh test_drop_hive_partition_and_refresh_e85660d6.partition_test_table partition (y=333, z=5309);
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
08:57:44
08:57:44 --------------------------- Captured stderr teardown ---------------------------
08:57:44 -- executing against localhost:21000
08:57:44 use default;
08:57:44
08:57:44 =================================== FAILURES ===================================
08:57:44 TestTpchMemLimitError.test_low_mem_limit_q9[mem_limit: 140 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
08:57:44 [gw2] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
08:57:44 query_test/test_mem_usage_scaling.py:158: in test_low_mem_limit_q9
08:57:44 xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
08:57:44 query_test/test_mem_usage_scaling.py:100: in low_memory_limit_test
08:57:44 assert TestLowMemoryLimits.EXPECTED_ERROR_MSG in str(e)
08:57:44 E assert 'Memory limit exceeded' in "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes"
08:57:44 E + where 'Memory limit exceeded' = TestLowMemoryLimits.EXPECTED_ERROR_MSG
08:57:44 E + and "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes" = str(ImpalaBeeswaxException())
08:57:44 ----------------------------- Captured stderr call -----------------------------
08:57:44 -- executing against localhost:21000
08:57:44 use tpch_parquet;
08:57:44
08:57:44 SET batch_size=0;
08:57:44 SET num_nodes=0;
08:57:44 SET disable_codegen=False;
08:57:44 SET abort_on_error=1;
08:57:44 SET mem_limit=140m;
08:57:44 SET exec_single_node_rows_threshold=0;
08:57:44 MainThread: Query Name:
08:57:44 TPCH-Q9
08:57:44
08:57:44 -- executing against localhost:21000
08:57:44 select
08:57:44 nation,
08:57:44 o_year,
08:57:44 sum(amount) as sum_profit
08:57:44 from(
08:57:44 select
08:57:44 n_name as nation,
08:57:44 year(o_orderdate) as o_year,
08:57:44 l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount
08:57:44 from
08:57:44 part,
08:57:44 supplier,
08:57:44 lineitem,
08:57:44 partsupp,
08:57:44 orders,
08:57:44 nation
08:57:44 where
08:57:44 s_suppkey = l_suppkey
08:57:44 and ps_suppkey = l_suppkey
08:57:44 and ps_partkey = l_partkey
08:57:44 and p_partkey = l_partkey
08:57:44 and o_orderkey = l_orderkey
08:57:44 and s_nationkey = n_nationkey
08:57:44 and p_name like '%green%'
08:57:44 ) as profit
08:57:44 group by
08:57:44 nation,
08:57:44 o_year
08:57:44 order by
08:57:44 nation,
08:57:44 o_year desc;
08:57:44
08:57:44 TestTpchMemLimitError.test_low_mem_limit_q9[mem_limit: 180 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none]
08:57:44 [gw2] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
08:57:44 query_test/test_mem_usage_scaling.py:158: in test_low_mem_limit_q9
08:57:44 xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
08:57:44 query_test/test_mem_usage_scaling.py:100: in low_memory_limit_test
08:57:44 assert TestLowMemoryLimits.EXPECTED_ERROR_MSG in str(e)
08:57:44 E assert 'Memory limit exceeded' in 'ImpalaBeeswaxException:\n Not connected'
08:57:44 E + where 'Memory limit exceeded' = TestLowMemoryLimits.EXPECTED_ERROR_MSG
08:57:44 E + and 'ImpalaBeeswaxException:\n Not connected' = str(ImpalaBeeswaxException())