Details

      Description

      We've seen two crashes in Jenkins builds with similar symptoms:

      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/main.py", line 84, in wrap_session
      08:57:44 INTERNALERROR>     doit(config, session)
      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/main.py", line 122, in _main
      08:57:44 INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
      08:57:44 Traceback (most recent call last):
      08:57:44   File "/usr/lib64/python2.6/logging/__init__.py", line 797, in emit
      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 521, in __call__
      08:57:44 INTERNALERROR>     return self._docall(self.methods, kwargs)
      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 528, in _docall
      08:57:44 INTERNALERROR>     firstresult=self.firstresult).execute()
      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/core.py", line 394, in execute
      08:57:44 INTERNALERROR>     res = method(*args)
      08:57:44 INTERNALERROR>   File "<remote exec>", line 47, in pytest_runtestloop
      08:57:44 INTERNALERROR>   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/execnet/gateway_base.py", line 737, in receive
      08:57:44 INTERNALERROR>     raise self._getremoteerror() or EOFError()
      08:57:44 INTERNALERROR> EOFError
      08:57:44     stream.write(fs % msg)
      08:57:44   File "/data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/infra/python/env/lib/python2.6/site-packages/_pytest/capture.py", line 229, in write
      08:57:44     self.buffer.write(obj)
      08:57:44 ValueError: I/O operation on closed file
      08:57:44 
      08:57:44 
      08:57:44 ==================================== ERRORS ====================================
      08:57:44  ERROR at teardown of TestRefreshPartition.test_drop_hive_partition_and_refresh[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none] 
      08:57:44 [gw1] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
      08:57:44 conftest.py:270: in cleanup
      08:57:44     request.instance.execute_query_expect_success(request.instance.client, "use default")
      08:57:44 common/impala_test_suite.py:433: in wrapper
      08:57:44     return function(*args, **kwargs)
      08:57:44 common/impala_test_suite.py:440: in execute_query_expect_success
      08:57:44     result = cls.__execute_query(impalad_client, query, query_options)
      08:57:44 common/impala_test_suite.py:525: in __execute_query
      08:57:44     return impalad_client.execute(query, user=user)
      08:57:44 common/impala_connection.py:160: in execute
      08:57:44     return self.__beeswax_client.execute(sql_stmt, user=user)
      08:57:44 beeswax/impala_beeswax.py:173: in execute
      08:57:44     handle = self.__execute_query(query_string.strip(), user=user)
      08:57:44 beeswax/impala_beeswax.py:337: in __execute_query
      08:57:44     handle = self.execute_query_async(query_string, user=user)
      08:57:44 beeswax/impala_beeswax.py:333: in execute_query_async
      08:57:44     return self.__do_rpc(lambda: self.imp_service.query(query,))
      08:57:44 beeswax/impala_beeswax.py:465: in __do_rpc
      08:57:44     raise ImpalaBeeswaxException(self.__build_error_message(u), u)
      08:57:44 E   ImpalaBeeswaxException: ImpalaBeeswaxException:
      08:57:44 E    INNER EXCEPTION: <class 'socket.error'>
      08:57:44 E    MESSAGE: [Errno 104] Connection reset by peer
      08:57:44 ---------------------------- Captured stderr setup -----------------------------
      08:57:44 SET sync_ddl=False;
      08:57:44 -- executing against localhost:21000
      08:57:44 DROP DATABASE IF EXISTS `test_drop_hive_partition_and_refresh_e85660d6` CASCADE;
      08:57:44 
      08:57:44 SET sync_ddl=False;
      08:57:44 -- executing against localhost:21000
      08:57:44 CREATE DATABASE `test_drop_hive_partition_and_refresh_e85660d6`;
      08:57:44 
      08:57:44 MainThread: Created database "test_drop_hive_partition_and_refresh_e85660d6" for test ID "metadata/test_refresh_partition.py::TestRefreshPartition::()::test_drop_hive_partition_and_refresh[exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: text/none]"
      08:57:44 ----------------------------- Captured stderr call -----------------------------
      08:57:44 -- executing against localhost:21000
      08:57:44 create table test_drop_hive_partition_and_refresh_e85660d6.partition_test_table (x int) partitioned by (y int, z int);
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 alter table test_drop_hive_partition_and_refresh_e85660d6.partition_test_table add partition (y=333, z=5309);
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 refresh test_drop_hive_partition_and_refresh_e85660d6.partition_test_table partition (y=333, z=5309);
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 show partitions test_drop_hive_partition_and_refresh_e85660d6.partition_test_table;
      08:57:44 
      08:57:44 --------------------------- Captured stderr teardown ---------------------------
      08:57:44 -- executing against localhost:21000
      08:57:44 use default;
      08:57:44 
      08:57:44 =================================== FAILURES ===================================
      08:57:44  TestTpchMemLimitError.test_low_mem_limit_q9[mem_limit: 140 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none] 
      08:57:44 [gw2] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
      08:57:44 query_test/test_mem_usage_scaling.py:158: in test_low_mem_limit_q9
      08:57:44     xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
      08:57:44 query_test/test_mem_usage_scaling.py:100: in low_memory_limit_test
      08:57:44     assert TestLowMemoryLimits.EXPECTED_ERROR_MSG in str(e)
      08:57:44 E   assert 'Memory limit exceeded' in "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes"
      08:57:44 E    +  where 'Memory limit exceeded' = TestLowMemoryLimits.EXPECTED_ERROR_MSG
      08:57:44 E    +  and   "ImpalaBeeswaxException:\n INNER EXCEPTION: <class 'thrift.transport.TTransport.TTransportException'>\n MESSAGE: TSocket read 0 bytes" = str(ImpalaBeeswaxException())
      08:57:44 ----------------------------- Captured stderr call -----------------------------
      08:57:44 -- executing against localhost:21000
      08:57:44 use tpch_parquet;
      08:57:44 
      08:57:44 SET batch_size=0;
      08:57:44 SET num_nodes=0;
      08:57:44 SET disable_codegen=False;
      08:57:44 SET abort_on_error=1;
      08:57:44 SET mem_limit=140m;
      08:57:44 SET exec_single_node_rows_threshold=0;
      08:57:44 MainThread: Query Name: 
      08:57:44 TPCH-Q9
      08:57:44 
      08:57:44 -- executing against localhost:21000
      08:57:44 select
      08:57:44   nation,
      08:57:44   o_year,
      08:57:44   sum(amount) as sum_profit
      08:57:44 from(
      08:57:44   select
      08:57:44     n_name as nation,
      08:57:44     year(o_orderdate) as o_year,
      08:57:44     l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount
      08:57:44   from
      08:57:44     part,
      08:57:44     supplier,
      08:57:44     lineitem,
      08:57:44     partsupp,
      08:57:44     orders,
      08:57:44     nation
      08:57:44   where
      08:57:44     s_suppkey = l_suppkey
      08:57:44     and ps_suppkey = l_suppkey
      08:57:44     and ps_partkey = l_partkey
      08:57:44     and p_partkey = l_partkey
      08:57:44     and o_orderkey = l_orderkey
      08:57:44     and s_nationkey = n_nationkey
      08:57:44     and p_name like '%green%'
      08:57:44   ) as profit
      08:57:44 group by
      08:57:44   nation,
      08:57:44   o_year
      08:57:44 order by
      08:57:44   nation,
      08:57:44   o_year desc;
      08:57:44 
      08:57:44  TestTpchMemLimitError.test_low_mem_limit_q9[mem_limit: 180 | exec_option: {'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} | table_format: parquet/none] 
      08:57:44 [gw2] linux2 -- Python 2.6.6 /data/jenkins/workspace/impala-umbrella-build-and-test/repos/Impala/bin/../infra/python/env/bin/python
      08:57:44 query_test/test_mem_usage_scaling.py:158: in test_low_mem_limit_q9
      08:57:44     xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
      08:57:44 query_test/test_mem_usage_scaling.py:100: in low_memory_limit_test
      08:57:44     assert TestLowMemoryLimits.EXPECTED_ERROR_MSG in str(e)
      08:57:44 E   assert 'Memory limit exceeded' in 'ImpalaBeeswaxException:\n Not connected'
      08:57:44 E    +  where 'Memory limit exceeded' = TestLowMemoryLimits.EXPECTED_ERROR_MSG
      08:57:44 E    +  and   'ImpalaBeeswaxException:\n Not connected' = str(ImpalaBeeswaxException())
      
      

        Attachments

          Activity

            People

            • Assignee:
              Unassigned
              Reporter:
              twmarshall Thomas Tauber-Marshall
            • Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: