[IMPALA-2925] Flaky test: test_alloc_fail

• Type: Bug
• Status: Resolved
• Priority: Minor
• Resolution: Cannot Reproduce
• Affects Version/s: Impala 2.5.0, Impala 2.8.0
• Fix Version/s: Impala 2.5.0
• Component/s: Backend
• Labels: broken-build test_issue
• Target Version: Impala 2.5.0
• Assignee: Michael Ho
• Reporter: Henry Robinson
• Created: 02/Feb/16 23:51
• Updated: 20/Sep/17 17:14
• Resolved: 20/Sep/17 17:14

Description

Saw this in a recent GVM for the final patch for IMPALA-1599; this failure seems unrelated:

http://sandbox.jenkins.cloudera.com/job/impala-external-gerrit-verify-merge/1716/testReport/junit/Impala.tests.custom_cluster.test_alloc_fail/TestAllocFail/test_alloc_fail_update_exec_option____disable_codegen___False___abort_on_error___1___exec_single_node_rows_threshold___0___batch_size___0___num_nodes___0____table_format__text_none_/

Error Message

assert 'failed to allocate' == ''   - failed to allocate
Stacktrace

self = <test_alloc_fail.TestAllocFail object at 0x1a15890>
vector = <tests.common.test_vector.TestVector object at 0x2852090>

    @pytest.mark.execute_serially
    @CustomClusterTestSuite.with_args("--stress_free_pool_alloc=3")
    def test_alloc_fail_update(self, vector):
>     self.run_test_case('QueryTest/alloc-fail-update', vector)

custom_cluster/test_alloc_fail.py:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <test_alloc_fail.TestAllocFail object at 0x1a15890>
test_file_name = 'QueryTest/alloc-fail-update'
vector = <tests.common.test_vector.TestVector object at 0x2852090>
use_db = None, multiple_impalad = False, encoding = None

    def run_test_case(self, test_file_name, vector, use_db=None, multiple_impalad=False,
        encoding=None):
      """
        Runs the queries in the specified test based on the vector values
    
        Runs the query using targeting the file format/compression specified in the test
        vector and the exec options specified in the test vector. If multiple_impalad=True
        a connection to a random impalad will be chosen to execute each test section.
        Otherwise, the default impalad client will be used.
        Additionally, the encoding for all test data can be specified using the 'encoding'
        parameter. This is useful when data is ingested in a different encoding (ex.
        latin). If not set, the default system encoding will be used.
        """
      table_format_info = vector.get_value('table_format')
      exec_options = vector.get_value('exec_option')
    
      # Resolve the current user's primary group name.
      group_id = pwd.getpwnam(getuser()).pw_gid
      group_name = grp.getgrgid(group_id).gr_name
    
      target_impalad_clients = list()
      if multiple_impalad:
        target_impalad_clients =\
            map(ImpalaTestSuite.create_impala_client, IMPALAD_HOST_PORT_LIST)
      else:
        target_impalad_clients = [self.client]
    
      # Change the database to reflect the file_format, compression codec etc, or the
      # user specified database for all targeted impalad.
      for impalad_client in target_impalad_clients:
        ImpalaTestSuite.change_database(impalad_client,
            table_format_info, use_db, pytest.config.option.scale_factor)
        impalad_client.set_configuration(exec_options)
    
      sections = self.load_query_test_file(self.get_workload(), test_file_name,
          encoding=encoding)
      for test_section in sections:
        if 'QUERY' not in test_section:
          assert 0, 'Error in test file %s. Test cases require a -- QUERY section.\n%s' %\
              (test_file_name, pprint.pformat(test_section))
    
        if 'SETUP' in test_section:
          self.execute_test_case_setup(test_section['SETUP'], table_format_info)
    
        # TODO: support running query tests against different scale factors
        query = QueryTestSectionReader.build_query(test_section['QUERY']
            .replace('$GROUP_NAME', group_name)
            .replace('$IMPALA_HOME', IMPALA_HOME)
            .replace('$FILESYSTEM_PREFIX', FILESYSTEM_PREFIX))
    
        if 'QUERY_NAME' in test_section:
          LOG.info('Query Name: \n%s\n' % test_section['QUERY_NAME'])
    
        # Support running multiple queries within the same test section, only verifying the
        # result of the final query. The main use case is to allow for 'USE database'
        # statements before a query executes, but it is not limited to that.
        # TODO: consider supporting result verification of all queries in the future
        result = None
        target_impalad_client = choice(target_impalad_clients)
        query_options_changed = []
        try:
          user = None
          if 'USER' in test_section:
            # Create a new client so the session will use the new username.
            user = test_section['USER'].strip()
            target_impalad_client = self.create_impala_client()
          for query in query.split(';'):
            set_pattern_match = SET_PATTERN.match(query)
            if set_pattern_match != None:
              query_options_changed.append(set_pattern_match.groups()[0])
            result = self.__execute_query(target_impalad_client, query, user=user)
        except Exception as e:
          if 'CATCH' in test_section:
            # In error messages, some paths are always qualified and some are not.
            # So, allow both $NAMENODE and $FILESYSTEM_PREFIX to be used in CATCH.
            expected_str = test_section['CATCH'].strip() \
                .replace('$FILESYSTEM_PREFIX', FILESYSTEM_PREFIX) \
                .replace('$NAMENODE', NAMENODE) \
                .replace('$IMPALA_HOME', IMPALA_HOME)
            assert expected_str in str(e)
            continue
          raise
        finally:
          if len(query_options_changed) > 0:
            self.restore_query_options(query_options_changed)
    
        if 'CATCH' in test_section:
>         assert test_section['CATCH'].strip() == ''
E         assert 'failed to allocate' == ''
E           - failed to allocate

Issue Links

• blocks: IMPALA-3229 Allow all contributors (or at least committers?) to run GVM