Index: testutils/ptest/Buffer.py =================================================================== --- /dev/null +++ testutils/ptest/Buffer.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +class Buffer(): + def __init__(self, size = 100, abandon_output = True): + self.buf_short = collections.deque(maxlen = size) + self.buf = [] + self.abandon_output = abandon_output + + def put(self, line): + if not self.abandon_output: + self.buf.append(line) + self.buf_short.append(line) + + def get_short(self): + return ''.join(self.buf_short) + + def get_long(self): + if self.abandon_output: + return None + else: + return ''.join(self.buf) Index: testutils/ptest/Process.py =================================================================== --- /dev/null +++ testutils/ptest/Process.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +from subprocess import Popen, PIPE, STDOUT +from shlex import split + +from Buffer import Buffer + +def run(cmd, quiet = False, abandon_output = True): + proc = Popen(split(cmd), stdout = PIPE, stderr = STDOUT) + buf = Buffer(abandon_output = abandon_output) + line = proc.stdout.readline() + while len(line): + buf.put(line) + if not quiet: + print(line, end = '') + line = proc.stdout.readline() + # Process could probably close the descriptor before exiting. + proc.wait() + if proc.returncode != 0: + raise Exception('Process exited with a non-zero return code. ' + + 'Last output of the program:\n\n' + + '---------- Start of exception log --\n' + + buf.get_short().strip() + + '\n---------- End of exception log --\n') + return buf.get_long() Index: testutils/ptest/README =================================================================== --- /dev/null +++ testutils/ptest/README @@ -0,0 +1,49 @@ += Configuration file = + +Configuration file is JSON formated, example: + +{ + "qfile_hosts": [ + ["hostname1", 2], + ["hostname2", 4], + ["hostname3", 4], + ], + "other_hosts": [ + ["hostname1", 2], + ["hostname4", 5] + ], + "master_base_path": "${{HOME}}/hivetests", + "host_base_path": "/mnt/drive/hivetests" + "java_home": "/opt/jdk" +} + +== qfile_hosts == +List of hosts that should run TestCliDriver and TestNegativeCliDriver test +cases. Number following the host name is number of simultaneous tests that +should be run on this host, you should probably set it near number of cores that +host has. + +== other_hosts == +List of hosts that should run all other test cases. Number has the same meaning +as in `qfile_hosts`. + +== master_base_path == +Path on localhost (master) where this script can build Hive, store reports, etc. +This path should be available from every slave node and should point to the same +data (home on NFS would be a good choice). + +== host_base_path == +Path on slaves where Hive repo will be cloned and tests will be run. +'-your_user_name' will be actually appended to this path to allow parallel runs +by different users. + +== java_home == +Should point to Java environment that should be used. + +== About paths == +You can use environmental variables with `${{my_env}}`, as home is used in the +example. + +You shouldn't point this paths to your work repository or any directory that +stores data you don't want to lose. This script might wipe everything under +`master_base_path` and `host_base_path` as needed. Index: testutils/ptest/Report.py =================================================================== --- /dev/null +++ testutils/ptest/Report.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +import os.path +import re +import base64 +import argparse +from xml.dom import Node +from xml.dom.minidom import parseString + +from mako.template import Template + +report_dir = os.path.dirname(os.path.realpath(__file__)) +one_file = False + +class TemplateRenderer(): + def __init__(self, template): + self.__template = template + + def render(self, file_name = None): + if file_name is None: + file_name = self.__template + if not file_name.startswith('/'): + current_dir = os.path.dirname(os.path.realpath(__file__)) + file_name = os.path.join(current_dir, file_name) + return Template(filename = file_name).render(this = self) + + def render_link(self, link_name, file_name): + if one_file: + return 'data:text/html;charset=utf-8;base64,' + \ + base64.b64encode(self.render(file_name)) + else: + part = self.render(file_name) + with open(report_dir + '/' + link_name, 'w') as f: + f.write(part) + return link_name + + def render_files(self): + report = self.render() + with open(report_dir + '/report.html', 'w') as f: + f.write(report) + +class TestCase(TemplateRenderer): + def __init__(self, element): + TemplateRenderer.__init__(self, 'templates/TestCase.html') + + self.__class_name = element.getAttribute('classname') + self.__name = element.getAttribute('name') + self.__time = float(element.getAttribute('time')) + self.__failure = False + self.__error = False + self.__log = None + + for child in element.childNodes: + if child.nodeType == Node.ELEMENT_NODE and child.tagName == 'failure': + self.__failure = True + self.__log = child.firstChild.nodeValue + elif child.nodeType == Node.ELEMENT_NODE and child.tagName == 'error': + self.__error = True + self.__log = child.firstChild.nodeValue + + def success(self): + return not (self.failure() or self.error()) + + def failure(self): + return self.__failure + + def error(self): + return self.__error + + def get_log(self): + return self.__log + + def get_name(self): + return self.__name + + def get_time(self): + return self.__time + +class TestSuite(TemplateRenderer): + def __init__(self, text): + TemplateRenderer.__init__(self, 'templates/TestSuite.html') + + self.properties = {} + self.test_cases = [] + + xml = parseString(text) + self.__populate_properties(xml) + self.__populate_test_cases(xml) + + top = xml.getElementsByTagName('testsuite')[0] + self.__errors = int(top.getAttribute('errors')) + self.__failures = int(top.getAttribute('failures')) + self.__tests = int(top.getAttribute('tests')) + self.__host_name = top.getAttribute('hostname') + dist_dir = self.properties['dist.dir'] + build_number = re.findall(self.__host_name + '-([0-9]+)$', dist_dir) + if build_number: + # Multiple builds per host. + self.__host_name += '-' + build_number[0] + self.__name = top.getAttribute('name').split('.')[-1] + self.__time = float(top.getAttribute('time')) + + def __populate_properties(self, xml): + properties = xml.getElementsByTagName('property') + for prop in properties: + self.properties[prop.getAttribute('name')] = prop.getAttribute('value') + + def __populate_test_cases(self, xml): + test_cases = xml.getElementsByTagName('testcase') + for test in test_cases: + self.test_cases.append(TestCase(test)) + + def tests(self): + return self.__tests + + def failures(self): + return self.__failures + + def errors(self): + return self.__errors + + def passes(self): + return self.tests() - self.failures() - self.errors() + + def time(self): + return self.__time + + def host_name(self): + return self.__host_name + + def name(self): + return self.__name + + def label(self): + return self.host_name() + '-' + self.name() + +class TestRun(TemplateRenderer): + def __init__(self, pwd): + TemplateRenderer.__init__(self, 'templates/TestRun.html') + + self.test_suites = [] + + files = os.listdir(pwd) + pattern = re.compile('^TEST-.*\.xml$') + for f in files: + if pattern.search(f) is not None: + with open(os.path.join(pwd, f)) as handle: + self.test_suites.append(TestSuite(handle.read())) + + def passes(self): + return reduce(lambda acc, x: acc + x.passes(), self.test_suites, 0) + + def failures(self): + return reduce(lambda acc, x: acc + x.failures(), self.test_suites, 0) + + def errors(self): + return reduce(lambda acc, x: acc + x.errors(), self.test_suites, 0) + + def tests(self): + return reduce(lambda acc, x: acc + x.tests(), self.test_suites, 0) + + def time(self): + return reduce(lambda acc, x: acc + x.time(), self.test_suites, 0) + + def success_rate(self): + if self.tests(): + return 100.0 * self.passes() / self.tests() + else: + return 100.0 + +def make_report(args): + global report_dir, one_file + report_dir = args.report_dir + one_file = args.one_file + + test_run = TestRun(args.log_dir) + test_run.render_files() + print('Summary:') + print(' tests run:', test_run.tests()) + print(' failures:', test_run.failures()) + print(' errors:', test_run.errors()) + + failed_results = {} + files = os.listdir(args.log_dir) + pattern = re.compile('^([^-]+-[^-]+)-(.*)\.fail$') + for f in files: + match = pattern.findall(f) + if match: + (host, test, ) = match[0] + if host not in failed_results: + failed_results[host] = [] + failed_results[host].append(test) + if failed_results: + print() + print('Some tests faled to produce a log and are not included in the report:') + for host in failed_results: + print(' {0}:'.format(host)) + for test in failed_results[host]: + print(' {0}'.format(test)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description = 'Create HTML report from JUnit logs.') + parser.add_argument(dest = 'log_dir', + help = 'Path to directory containing JUnit logs') + parser.add_argument(dest = 'report_dir', + help = 'Where should the report be generated') + parser.add_argument('--one-file', action = 'store_true', dest = 'one_file', + help = 'Inline everything and generate only one file') + args = parser.parse_args() + + make_report(args) Index: testutils/ptest/Ssh.py =================================================================== --- /dev/null +++ testutils/ptest/Ssh.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Thread +from Queue import Queue + +import Process + +class SSHConnection(): + def __init__(self, host, num = None): + self.host = host + if num is None: + self.hostname = host + else: + self.hostname = host + '-' + str(num); + self.pwd = '/' + self.env = {} + self.path = [] + + def cd(self, path): + self.pwd = path.format(host = self.hostname) + + def export(self, env, value): + self.env[env] = value.format(host = self.hostname) + + def add_path(self, path): + self.path.append(path.format(host = self.hostname)) + + def prefix(self, cmd): + pre = [] + pre.append('cd "{0}"'.format(self.pwd)) + for (e, v) in self.env.iteritems(): + pre.append('export {0}="{1}"'.format(e, v)) + for p in self.path: + pre.append('export PATH="{0}:${{PATH}}"'.format(p)) + pre.append(cmd) + return ' && '.join(pre) + + def run(self, cmd, warn_only = False, quiet = False, vewy_quiet = False, + abandon_output = True): + # Don't use single quotes in `cmd`, this will break and end badly. + cmd = cmd.format(host = self.hostname) + cmd = self.prefix(cmd) + print(self.hostname + ' =>') + if vewy_quiet: + # Be vewy, vewy quiet, I'm hunting wabbits. + print('[command hidden]\n') + quiet = True + else: + print(cmd + '\n') + cmd = "ssh -nT '{0}' '{1}'".format(self.host, cmd) + try: + return Process.run(cmd, quiet, abandon_output) + except Exception as e: + if warn_only: + print(str(e) + '---------- This was only a warning, ' + + 'it won\'t stop the execution --\n') + return None + else: + raise e + +class SSHSet(): + def __init__(self, conn = []): + self.conn = conn + + def __len__(self): + return len(self.conn) + + def add(self, conn): + if isinstance(conn, list): + self.conn.extend(conn) + else: + self.conn.append(conn) + + def cd(self, path): + for conn in self.conn: + conn.cd(path) + + def export(self, env, value): + for conn in self.conn: + conn.export(env, value) + + def add_path(self, path): + for conn in self.conn: + conn.add_path(path) + + def run(self, cmd, parallel = True, quiet = False, vewy_quiet = False, + abandon_output = True, warn_only = False): + if not parallel: + for conn in self.conn: + conn.run(cmd, quiet = quiet, vewy_quiet = vewy_quiet, + abandon_output = abandon_output, warn_only = warn_only) + else: + threads = [] + queue = Queue() + def wrapper(conn, cmd, queue): + try: + conn.run(cmd, quiet = quiet, vewy_quiet = vewy_quiet, + abandon_output = abandon_output, + warn_only = warn_only) + except Exception as e: + queue.put(Exception(conn.hostname + ' => ' + str(e))) + for conn in self.conn: + thread = Thread(target = wrapper, args = (conn, cmd, queue, )) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + if not queue.empty(): + l = [] + while not queue.empty(): + e = queue.get() + l.append(str(e)); + raise Exception('\n'.join(l)) Index: testutils/ptest/config.py =================================================================== --- /dev/null +++ testutils/ptest/config.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import json +import copy +import getpass +import os.path + +from Ssh import SSHConnection, SSHSet + +local = None +qfile_set = None +other_set = None +remote_set = None +all_set = None + +master_base_path = None +host_base_path = None +java_home = None + +def load(config_file = '~/.hive_ptest.conf'): + global local, qfile_set, other_set, remote_set, all_set + global master_base_path, host_base_path, java_home + + config_file = os.path.expanduser(config_file) + + cfg = None + try: + with open(config_file) as f: + cfg = json.loads(f.read()) + + host_nodes = {} + def get_node(host): + if not host in host_nodes: + host_nodes[host] = -1 + host_nodes[host] += 1 + return SSHConnection(host, host_nodes[host]) + + qfile = [] + for (host, num, ) in cfg['qfile_hosts']: + for i in range(num): + qfile.append(get_node(host)) + + other = [] + for (host, num, ) in cfg['other_hosts']: + for i in range(num): + other.append(get_node(host)) + + local = SSHConnection('localhost') + + qfile_set = SSHSet(qfile) + other_set = SSHSet(other) + + # Make copies, otherwise they they will be passed by reference and + # reused. Reuse is bad - you don't want `cd` on remote_set to affect + # anything in the all_set. + + remote_set = SSHSet(copy.copy(qfile)) + remote_set.add(copy.copy(other)) + + all_set = SSHSet(copy.copy(qfile)) + all_set.add(copy.copy(other)) + all_set.add(local) + + master_base_path = cfg['master_base_path'] + host_base_path = cfg['host_base_path'] + '-' + getpass.getuser() + java_home = cfg['java_home'] + except Exception as e: + raise Exception('Failed to parse your configuration file (' + + config_file + '). Maybe you forgot the `--config` switch?', e) Index: testutils/ptest/hivetest.py =================================================================== --- /dev/null +++ testutils/ptest/hivetest.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import time +from threading import Thread +import os.path +import collections +import re + +import Report +import config + +# WARNING +# +# If you are editing this code, please be aware that commands passed to `run` +# should not use single quotes, this will break and end badly as the final +# command looks like `ssh 'host' 'some command - single quote will break it'`. +# Also please be aware that `run` uses `.format` to change `{host}` in commands +# into actual host name it is running on, running `.format` on strings using +# `{host}`, for example including `host_code_path` will not work. +# +# Also this code assumes master_base_path is available to all testing machines +# and is mounted in the same place on all of them. +# +# Getting rid of this restrictions without making the code much more complicated +# is very welcome. + +# This is configured in user configuration file. + +local = None +qfile_set = None +other_set = None +remote_set = None +all_set = None + +master_base_path = None +host_base_path = None + +# End of user configurated things. + +ant_path = None +arc_path = None +phutil_path = None +code_path = None +report_path = None +host_code_path = None + +def read_conf(config_file): + global local, qfile_set, other_set, remote_set, all_set + global master_base_path, host_base_path + global ant_path, arc_path, phutil_path, code_path, report_path, host_code_path + + if config_file is not None: + config.load(config_file) + else: + config.load() + + local = config.local + qfile_set = config.qfile_set + other_set = config.other_set + remote_set = config.remote_set + all_set = config.all_set + + master_base_path = config.master_base_path + host_base_path = config.host_base_path + + ant_path = master_base_path + '/apache-ant-1.8.2' + arc_path = master_base_path + '/arcanist' + phutil_path = master_base_path + '/libphutil' + code_path = master_base_path + '/trunk' + report_path = master_base_path + '/report/' + time.strftime('%m.%d.%Y_%H:%M:%S') + host_code_path = host_base_path + '/trunk-{host}' + + # Setup of needed environmental variables and paths + + # Ant + all_set.add_path(ant_path + '/bin') + + # Arcanist + all_set.add_path(arc_path + '/bin') + + # Java + all_set.export('JAVA_HOME', config.java_home) + all_set.add_path(config.java_home + '/bin') + + # Hive + remote_set.export('HIVE_HOME', host_code_path + '/build/dist') + remote_set.add_path(host_code_path + '/build/dist/bin') + + # Hadoop + remote_set.export('HADOOP_HOME', host_code_path + + '/build/hadoopcore/hadoop-0.20.1') + +def get_ant(): + # Gets Ant 1.8.2 from one of Apache mirrors. + print('\n-- Installing Ant 1.8.2\n') + + if local.run('test -d "{0}"'.format(ant_path), warn_only = True, + abandon_output = False) is None: + local.cd(master_base_path) + local.run('curl "http://apache.osuosl.org//ant/binaries/apache-ant-1.8.2-bin.tar.gz" | tar xz') + else: + print('\n Ant 1.8.2 already installed\n') + +def get_arc(): + # Gets latest Arcanist and libphtuil from their Git repositories. + print('\n-- Updating Arcanist installation\n') + + if local.run('test -d "{0}"'.format(arc_path), warn_only = True, + abandon_output = False) is None: + local.run('mkdir -p "{0}"'.format(os.path.dirname(arc_path))) + local.run('git clone git://github.com/facebook/arcanist.git "{0}"' + .format(arc_path)) + + if local.run('test -d "{0}"'.format(phutil_path), warn_only = True, + abandon_output = False) is None: + local.run('mkdir -p "{0}"'.format(os.path.dirname(phutil_path))) + local.run('git clone git://github.com/facebook/libphutil.git "{0}"' + .format(phutil_path)) + + local.cd(arc_path) + local.run('git pull') + local.cd(phutil_path) + local.run('git pull') + +def get_clean_hive(): + # Gets latest Hive from Apache Git repository and cleans the repository + # (undo of any changes and removal of all generated files). Also runs + # `arc-setup` so the repo is ready to be used. + print('\n-- Updating Hive repo\n') + + if local.run('test -d "{0}"'.format(code_path), warn_only = True, + abandon_output = False) is None: + local.run('mkdir -p "{0}"'.format(os.path.dirname(code_path))) + local.run('git clone git://git.apache.org/hive.git "{0}"'.format(code_path)) + + local.cd(code_path) + local.run('git reset --hard HEAD') + local.run('git clean -dffx') + local.run('git pull') + local.run('ant arc-setup') + +def prepare_for_reports(): + # Generates directories for test reports. All test nodes will copy results + # to this directories. + print('\n-- Creating a directory for JUnit reports\n') + # Remove previous reports that might be there. + local.run('rm -rf "{0}"'.format(report_path), warn_only = True) + local.run('mkdir -p "{0}/logs"'.format(report_path)) + local.run('mkdir -p "{0}/out/clientpositive"'.format(report_path)) + local.run('mkdir -p "{0}/out/clientnegative"'.format(report_path)) + +def patch_hive(patches = [], revision = None): + # Applies given patches to the Hive repo. Revision means a Differential + # revision, patches list is a list of paths to patches on local file system. + # + # Allowing multiple revisions and patches would complicate things a little + # (order of applied patches should be preserved, but argparse will split + # them into two lists) so only multiple local patches are allowed. + # Shouldn't be a big problem as you can use `arc export` to get the patches + # locally. + local.cd(code_path) + if revision is not None: + print('\n-- Patching Hive repo using a Differential revision\n') + revision = revision.upper() + if not revision.startswith('D'): + revision = 'D' + revision + local.run('arc patch "{0}"'.format(revision)) + if patches: + print('\n-- Patching Hive repo using a patch from local file system\n') + for patch in patches: + local.run('patch -f -p0 < "{0}"'.format(patch)) + +def build_hive(): + print('\n-- Building Hive\n') + local.cd(code_path) + local.run('ant package') + +def propagate_hive(): + # Expects master_base_path to be available on all test nodes in the same + # place (for example using NFS). + print('\n-- Propagating Hive repo to all hosts\n') + remote_set.run('mkdir -p "{0}"'.format(host_code_path)) + remote_set.run('rsync -qa --delete "{0}/" "{1}"'.format( + code_path, host_code_path)) + +def segment_tests(path): + # Removes `.q` files that should not be run on this host. The huge shell + # command is slow (not really suprising considering amount of forking it has + # to do), you are welcome to make it better=). + local.cd(code_path + path) + tests = local.run('ls -1', quiet = True, abandon_output = False).strip().split('\n') + + qfile_set.cd(host_code_path + path) + cmd = [] + i = 0 + for test in tests: + host = qfile_set.conn[i].hostname + cmd.append('if [[ "{host}" != "' + host + '" ]]; then rm -f "' + test + '"; fi') + i = (i + 1) % len(qfile_set) + cmd = ' && '.join(cmd) + # The command is huge and printing it out is not very useful, using wabbit + # hunting mode. + qfile_set.run(cmd, vewy_quiet = True) + +def prepare_tests(): + print('\n-- Preparing test sets on all hosts\n') + segment_tests('/ql/src/test/queries/clientpositive') + segment_tests('/ql/src/test/queries/clientnegative') + +def collect_log(name): + # Moves JUnit log to the global logs directory. + # + # This has the same restriction on master_base_path as propagate_hive. + new_name = name.split('.') + new_name[-2] += '-{host}' + new_name = '.'.join(new_name) + qfile_set.cd(host_code_path + '/build/ql/test') + # If tests failed there may be no file, so warn only if `cp` is having + # problems. + qfile_set.run( + 'cp "' + name + '" "' + report_path + '/logs/' + new_name + '" || ' + + 'touch "' + report_path + '/logs/{host}-' + name + '.fail"' + ) + # Get the hive.log too. + qfile_set.cd(host_code_path + '/build/ql/tmp') + qfile_set.run('cp "hive.log" "' + report_path + '/logs/hive-{host}-' + name + '.log"', + warn_only = True) + +def collect_out(name): + # Moves `.out` file (test output) to the global logs directory. + # + # This has the same restriction on master_base_path as propagate_hive. + qfile_set.cd(host_code_path + '/build/ql/test/logs/' + name) + # Warn only if no files are found. + qfile_set.run('cp * "' + report_path + '/out/' + name + '"', warn_only = True) + +def run_tests(): + # Runs TestCliDriver and TestNegativeCliDriver testcases. + print('\n-- Running .q file tests on all hosts\n') + + # Using `quiet` because output of `ant test` is not very useful when we are + # running on many hosts and it all gets mixed up. In case of an error + # you'll get last lines generated by `ant test` anyway (might be totally + # irrelevant if one of the first tests fails and Ant reports a failure after + # running all the other test, fortunately JUnit report saves the Ant output + # if you need it for some reason). + + qfile_set.cd(host_code_path) + qfile_set.run('ant -Dtestcase=TestCliDriver -Doffline=true test', + quiet = True, warn_only = True) + collect_log('TEST-org.apache.hadoop.hive.cli.TestCliDriver.xml') + collect_out('clientpositive') + + qfile_set.cd(host_code_path) + qfile_set.run('ant -Dtestcase=TestNegativeCliDriver -Doffline=true test', + quiet = True, warn_only = True) + collect_log('TEST-org.apache.hadoop.hive.cli.TestNegativeCliDriver.xml') + collect_out('clientnegative') + +def run_other_tests(): + # Runs all other tests that run_test doesn't run. + + def get_other_list(): + local.cd(code_path) + # Generate test classes in build. + local.run('ant -Dtestcase=nothing test') + tests = local.run(' | '.join([ + 'find build/*/test/classes -name "Test*.class"', + 'sed -e "s:[^/]*/::g"', + 'grep -v TestSerDe.class', + 'grep -v TestHiveMetaStore.class', + 'grep -v TestCliDriver.class', + 'grep -v TestNegativeCliDriver.class', + 'grep -v ".*\$.*\.class"', + 'sed -e "s:\.class::"' + ]), abandon_output = False) + return tests.split() + + def segment_other(): + # Split all test cases between hosts. + def get_command(test): + return '; '.join([ + 'ant -Dtestcase=' + test + ' -Doffline=true test', + + 'cp "`find . -name "TEST-*.xml"`" "' + report_path + '/logs/" || ' + + 'touch "' + report_path + '/logs/{host}-' + test + '.fail"', + + 'cp "build/ql/tmp/hive.log" "' + report_path + '/logs/hive-{host}-' + test + '.log"' + ]) + cmd = [] + i = 0 + for test in get_other_list(): + # Special case, don't run minimr tests in parallel. They will run + # on the first host, and no other tests will run there (unless we + # have a single host). + # + # TODO: Real fix would be to allow parallel runs of minimr tests. + if len(other_set) > 1: + if re.match('.*minimr.*', test.lower()): + host = other_set.conn[0].hostname + else: + i = (i + 1) % len(other_set) + if i == 0: + i = 1 + host = other_set.conn[i].hostname + else: + # We are running on single host. + host = other_set.conn[0].hostname + cmd.append( + 'if [[ "{host}" == "' + host + '" ]]; then ' + + get_command(test) + + '; fi' + ) + return ' ; '.join(cmd) + + command = segment_other() + other_set.cd(host_code_path) + # See comment about quiet option in run_tests. + other_set.run(command, quiet = True, warn_only = True) + +def generate_report(one_file_report = False): + # Uses `Report.py` to create a HTML report. + print('\n-- Generating a test report\n') + + # Call format to remove '{{' and '}}'. + path = os.path.expandvars(report_path.format()) + CmdArgs = collections.namedtuple('CmdArgs', ['one_file', 'log_dir', 'report_dir']) + args = CmdArgs( + one_file = one_file_report, + log_dir = '{0}/logs'.format(path), + report_dir = path + ) + Report.make_report(args) + + print('\n-- Test report has been generated and is available here:') + print('-- "{0}/report.html"'.format(path)) + print() + +def stop_tests(): + # Brutally stops tests on all hosts, something more subtle would be nice and + # would allow the same user to run this script multiple times + # simultaneously. + print('\n-- Stopping tests on all hosts\n') + remote_set.run('killall -9 java', warn_only = True) + +def remove_code(): + # Running this only on one connection per host so there are no conflicts + # between several `rm` calls. This removes all repositories, it would have + # to be changed if we were to allow multiple simultaneous runs of this + # script. + + print('\n-- Removing Hive code from all hosts\n') + # We could remove only `host_code_path`, but then we would have abandoned + # directories after lowering number of processes running on one host. + cmd = 'rm -rf "' + host_base_path + '"' + cmd = 'if [[ `echo "{host}" | grep -q -- "-0$"; echo "$?"` -eq "0" ]]; then ' + \ + cmd + '; fi' + remote_set.run(cmd) + +def overwrite_results(): + # Copy generated `.q.out` files to master repo. + + local.cd(code_path) + expanded_path = local.run('pwd', abandon_output = False) + print('\n-- Copying generated `.q.out` files to master repository: ' + + expanded_path) + + for name in ['clientpositive', 'clientnegative']: + local.cd(report_path + '/out/' + name) + # Don't panic if no files are found. + local.run('cp * "' + code_path + '/ql/src/test/results/' + name + '"', + warn_only = True) + +# -- Tasks that can be called from command line start here. + +def cmd_prepare(patches = [], revision = None): + get_ant() + get_arc() + get_clean_hive() + patch_hive(patches, revision) + build_hive() + propagate_hive() + prepare_tests() + +def cmd_run_tests(one_file_report = False): + t = Thread(target = run_other_tests) + t.start() + prepare_for_reports() + run_tests() + t.join() + + if args.overwrite: + overwrite_results() + + generate_report(one_file_report) + +def cmd_test(patches = [], revision = None, one_file_report = False): + cmd_prepare(patches, revision) + cmd_run_tests(one_file_report) + +def cmd_stop(): + stop_tests() + +def cmd_remove(): + remove_code() + +parser = argparse.ArgumentParser(description = + 'Hive test farm controller.') +parser.add_argument('--config', dest = 'config', + help = 'Path to configuration file') +parser.add_argument('--prepare', action = 'store_true', dest = 'prepare', + help = 'Builds Hive and propagates it to all test machines') +parser.add_argument('--run-tests', action = 'store_true', dest = 'run_tests', + help = 'Runs tests on all test machines') +parser.add_argument('--test', action = 'store_true', dest = 'test', + help = 'Same as running `prepare` and then `run-tests`') +parser.add_argument('--report-name', dest = 'report_name', + help = 'Store report and logs directory called `REPORT_NAME`') +parser.add_argument('--stop', action = 'store_true', dest = 'stop', + help = 'Kill misbehaving tests on all machines') +parser.add_argument('--remove', action = 'store_true', dest = 'remove', + help = 'Remove Hive trunk copies from test machines') +parser.add_argument('--revision', dest = 'revision', + help = 'Differential revision to test') +parser.add_argument('--patch', dest = 'patch', nargs = '*', + help = 'Patches from local file system to test') +parser.add_argument('--one-file-report', dest = 'one_file_report', + action = 'store_true', + help = 'Generate one (huge) report file instead of multiple small ones') +parser.add_argument('--overwrite', dest = 'overwrite', action = 'store_true', + help = 'Overwrite result files in master repo') +args = parser.parse_args() + +read_conf(args.config) + +if args.report_name: + report_path = '/'.join(report_path.split('/')[:-1] + [args.report_name]) + +if args.prepare: + cmd_prepare(args.patch, args.revision) +elif args.run_tests: + cmd_run_tests(args.one_file_report) +elif args.test: + cmd_test(args.patch, args.revision, args.one_file_report) +elif args.stop: + cmd_stop() +elif args.remove: + cmd_remove() Index: testutils/ptest/templates/Properties.html =================================================================== --- /dev/null +++ testutils/ptest/templates/Properties.html @@ -0,0 +1,41 @@ + + + + + ${this.host_name()} - Properties + + + + % if this.properties: + + + + + + % for (name, value, ) in this.properties.iteritems(): + + + + + % endfor +
NameValue
${name}${value}
+ % endif + + Index: testutils/ptest/templates/TestCase.html =================================================================== --- /dev/null +++ testutils/ptest/templates/TestCase.html @@ -0,0 +1,43 @@ + + + + ${this.get_name()} + + % if this.failure(): + Failed + % elif this.error(): + Error + % else: + Success + % endif + + ${this.get_time()}s + +% if this.failure() or this.error(): + + +
+${this.get_log()}
+      
+ + +% endif Index: testutils/ptest/templates/TestRun.html =================================================================== --- /dev/null +++ testutils/ptest/templates/TestRun.html @@ -0,0 +1,90 @@ + + + + + Unit Test Results. + + + +

Unit Test Results.

+ +

Summary

+ + + + + + + + + + + + + + + + + +
TestsPassesFailuresErrorsSuccess rateTime
${this.tests()}${this.passes()}${this.failures()}${this.errors()}${round(this.success_rate(), 2)}%${round(this.time(), 2)}s
+ + % if this.test_suites: +

Test results

+ + + + + + + + + + + % for test_suite in this.test_suites: + + + + + + + + + + % endfor +
HostNameTestsPassesFailuresErrorsTime
+ + ${test_suite.host_name()} + + ${test_suite.name()}${test_suite.tests()}${test_suite.passes()}${test_suite.failures()}${test_suite.errors()}${round(test_suite.time(), 2)}s
+ % endif + + % for test_suite in this.test_suites: + ${test_suite.render()} + % endfor + + Index: testutils/ptest/templates/TestSuite.html =================================================================== --- /dev/null +++ testutils/ptest/templates/TestSuite.html @@ -0,0 +1,37 @@ + + +

${this.label()}

+% if this.test_cases: + + + + + + + % for test_case in this.test_cases: + ${test_case.render()} + % endfor +
NameStatusTime
+% endif + + Properties > + +Back to top Index: testutils/ptest/templates/common.css =================================================================== --- /dev/null +++ testutils/ptest/templates/common.css @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +body { + font-size: 0.85em; +} + +a { + color: blue; +} + +table { + width: 100%; +} + +table.properties { + table-layout: fixed; +} + +table.properties th:first-child { + width: 30%; +} + +table.properties th:first-child + th { + width: 70%; +} + +table th { + background-color: #a6caf0; + padding: 0.3em; + text-align: left; +} + +table td { + background-color: #eeeee0; + padding: 0.3em; +} + +table tr.failure { + color: red; + font-weight: bold; +} + +table tr.failure a { + color: red; +} + +.long-lines { + word-wrap: break-word; +} + +.float-right { + float: right; +} + +.wide { + width: 100%; +}