From 0276c38f198c3c8a522c0b6193688b301bfed398 Mon Sep 17 00:00:00 2001 From: Tim Armstrong Date: Sun, 5 May 2019 11:35:03 -0700 Subject: [PATCH] WIP IMPALA-8508: download Python 2.7 from toolchain if needed This required refactoring the toolchain bootstrap logic so that the functions to download toolchain packages could be used by bootstrap_virtualenv. These functions are added to infra/python/lib/. TODO: * need to bump toolchain version * run linter * breaks on older OpenSSL versions Change-Id: Ic4ed59ef8aa1c317e37daa79ee3d195015727864 --- bin/bootstrap_toolchain.py | 154 +-------------------------- bin/impala-config.sh | 4 +- bin/set-pythonpath.sh | 1 + infra/python/bootstrap_virtualenv.py | 43 ++++++-- infra/python/lib/__init__.py | 1 + infra/python/lib/toolchain_util.py | 197 +++++++++++++++++++++++++++++++++++ infra/python/tmp.py | 17 +++ tests/common/impala_test_suite.py | 2 + 8 files changed, 263 insertions(+), 156 deletions(-) create mode 100644 infra/python/lib/__init__.py create mode 100644 infra/python/lib/toolchain_util.py create mode 100644 infra/python/tmp.py diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index f0b54f6..50479ca 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -42,65 +42,15 @@ import logging import multiprocessing.pool import os -import random -import re import sh import shutil import subprocess import sys import tempfile -import time - -from collections import namedtuple - -TOOLCHAIN_HOST = "https://native-toolchain.s3.amazonaws.com/build" - -# Maps return values from 'lsb_release -irs' to the corresponding OS labels for both the -# toolchain and the CDH components. -OsMapping = namedtuple('OsMapping', ['lsb_release', 'toolchain', 'cdh']) -OS_MAPPING = [ - OsMapping("centos5", "ec2-package-centos-5", None), - OsMapping("centos6", "ec2-package-centos-6", "redhat6"), - OsMapping("centos7", "ec2-package-centos-7", "redhat7"), - OsMapping("redhatenterpriseserver5", "ec2-package-centos-5", None), - OsMapping("redhatenterpriseserver6", "ec2-package-centos-6", "redhat6"), - OsMapping("redhatenterpriseserver7", "ec2-package-centos-7", "redhat7"), - OsMapping("debian6", "ec2-package-debian-6", None), - OsMapping("debian7", "ec2-package-debian-7", None), - OsMapping("debian8", "ec2-package-debian-8", "debian8"), - OsMapping("suselinux11", "ec2-package-sles-11", None), - OsMapping("suselinux12", "ec2-package-sles-12", "sles12"), - OsMapping("suse12.2", "ec2-package-sles-12", "sles12"), - OsMapping("suse12.3", "ec2-package-sles-12", "sles12"), - OsMapping("ubuntu12.04", "ec2-package-ubuntu-12-04", None), - OsMapping("ubuntu14.04", "ec2-package-ubuntu-14-04", None), - OsMapping("ubuntu15.04", "ec2-package-ubuntu-14-04", None), - OsMapping("ubuntu15.10", "ec2-package-ubuntu-14-04", None), - OsMapping('ubuntu16.04', "ec2-package-ubuntu-16-04", "ubuntu1604"), - OsMapping('ubuntu18.04', "ec2-package-ubuntu-18-04", "ubuntu1804") -] - -class Package(object): - """ - Represents a package to be downloaded. A version, if not specified - explicitly, is retrieved from the environment variable IMPALA__VERSION. - URLs are retrieved from IMPALA__URL, but are optional. - """ - def __init__(self, name, version=None, url=None): - self.name = name - self.version = version - self.url = url - package_env_name = name.replace("-", "_").upper() - if self.version is None: - version_env_var = "IMPALA_{0}_VERSION".format(package_env_name) - - self.version = os.environ.get(version_env_var) - if not self.version: - raise Exception("Could not find version for {0} in environment var {1}".format( - name, version_env_var)) - if self.url is None: - url_env_var = "IMPALA_{0}_URL".format(package_env_name) - self.url = os.environ.get(url_env_var) + +from toolchain_util import (check_output, download_package, get_platform_release_label, + get_toolchain_compiler, Package, package_directory, remove_existing_package, + TOOLCHAIN_HOST, try_get_platform_release_label, wget_and_unpack_package) class CdpComponent(object): @@ -123,82 +73,6 @@ class CdpComponent(object): pkg_directory if pkg_directory else basename) -def try_get_platform_release_label(): - """Gets the right package label from the OS version. Returns an OsMapping with both - 'toolchain' and 'cdh' labels. Return None if not found. - """ - try: - return get_platform_release_label() - except Exception: - return None - - -# Cache "lsb_release -irs" to avoid excessive logging from sh, and -# to shave a little bit of time. -lsb_release_cache = None - - -def get_platform_release_label(release=None): - """Gets the right package label from the OS version. Raise exception if not found. - 'release' can be provided to override the underlying OS version. - """ - global lsb_release_cache - if not release: - if lsb_release_cache: - release = lsb_release_cache - else: - release = "".join(map(lambda x: x.lower(), sh.lsb_release("-irs").split())) - # Only need to check against the major release if RHEL or CentOS - for platform in ['centos', 'redhatenterpriseserver']: - if platform in release: - release = release.split('.')[0] - break - lsb_release_cache = release - for mapping in OS_MAPPING: - if re.search(mapping.lsb_release, release): - return mapping - - raise Exception("Could not find package label for OS version: {0}.".format(release)) - -def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber): - if not download_path.endswith("/" + file_name): - raise Exception("URL {0} does not match with expected file_name {1}" - .format(download_path, file_name)) - NUM_ATTEMPTS = 3 - for attempt in range(1, NUM_ATTEMPTS + 1): - logging.info("Downloading {0} to {1}/{2} (attempt {3})".format( - download_path, destination, file_name, attempt)) - # --no-clobber avoids downloading the file if a file with the name already exists - try: - sh.wget(download_path, directory_prefix=destination, no_clobber=wget_no_clobber) - break - except Exception, e: - if attempt == NUM_ATTEMPTS: - raise - logging.error("Download failed; retrying after sleep: " + str(e)) - time.sleep(10 + random.random() * 5) # Sleep between 10 and 15 seconds. - logging.info("Extracting {0}".format(file_name)) - sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination) - sh.rm(os.path.join(destination, file_name)) - -def download_package(destination, package, compiler, platform_release=None): - remove_existing_package(destination, package.name, package.version) - - toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID"] - label = get_platform_release_label(release=platform_release).toolchain - format_params = {'product': package.name, 'version': package.version, - 'compiler': compiler, 'label': label, 'toolchain_build_id': toolchain_build_id} - file_name = "{product}-{version}-{compiler}-{label}.tar.gz".format(**format_params) - format_params['file_name'] = file_name - if package.url is None: - url_path = "/{toolchain_build_id}/{product}/{version}-{compiler}/{file_name}".format( - **format_params) - download_path = TOOLCHAIN_HOST + url_path - else: - download_path = package.url - - wget_and_unpack_package(download_path, file_name, destination, True) - def bootstrap(toolchain_root, packages): """Downloads and unpacks each package in the list `packages` into `toolchain_root` if it doesn't exist already. @@ -208,9 +82,7 @@ def bootstrap(toolchain_root, packages): check_custom_toolchain(toolchain_root, packages) return - # Detect the compiler - compiler = "gcc-{0}".format(os.environ["IMPALA_GCC_VERSION"]) - + compiler = get_toolchain_compiler() def handle_package(p): if check_for_existing_package(toolchain_root, p.name, p.version, compiler): return @@ -222,22 +94,6 @@ def bootstrap(toolchain_root, packages): get_platform_release_label().toolchain) execute_many(handle_package, packages) -def check_output(cmd_args): - """Run the command and return the output. Raise an exception if the command returns - a non-zero return code. Similar to subprocess.check_output() which is only provided - in python 2.7. - """ - process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdout, _ = process.communicate() - if process.wait() != 0: - raise Exception("Command with args '%s' failed with exit code %s:\n%s" - % (cmd_args, process.returncode, stdout)) - return stdout - -def package_directory(toolchain_root, pkg_name, pkg_version): - dir_name = "{0}-{1}".format(pkg_name, pkg_version) - return os.path.join(toolchain_root, dir_name) - def version_file_path(toolchain_root, pkg_name, pkg_version): return os.path.join(package_directory(toolchain_root, pkg_name, pkg_version), "toolchain_package_version.txt") diff --git a/bin/impala-config.sh b/bin/impala-config.sh index cc8cfef..0ede244 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -68,7 +68,7 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=24-3b615798c1 +export IMPALA_TOOLCHAIN_BUILD_ID=27-3ad209adbc # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p4 @@ -128,6 +128,8 @@ export IMPALA_PROTOBUF_VERSION=3.5.1 unset IMPALA_PROTOBUF_URL export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=42.2.5 unset IMPALA_POSTGRES_JDBC_DRIVER_URL +export IMPALA_PYTHON_VERSION=2.7.16 +unset IMPALA_PYTHON_URL export IMPALA_RAPIDJSON_VERSION=1.1.0 unset IMPALA_RAPIDJSON_URL export IMPALA_RE2_VERSION=20190301 diff --git a/bin/set-pythonpath.sh b/bin/set-pythonpath.sh index 86a5fe7..3dc2f96 100755 --- a/bin/set-pythonpath.sh +++ b/bin/set-pythonpath.sh @@ -25,6 +25,7 @@ else PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/shell/gen-py fi PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/testdata/ +PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/infra/python/lib # There should be just a single version of python that created the # site-packages directory. We find it by performing shell independent expansion diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 27c527f..9f5f672 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -47,6 +47,22 @@ import tempfile import textwrap import urllib +print("PATH IS: " + str(sys.path)) +try: + print("PYTHONPATH " + os.environ.get("PYTHONPATH")) +except: + pass + +# Ensure the ssl modules are loaded correctly. +import ssl +from httplib import HTTPConnection +from urllib2 import HTTPSHandler + +print("SUCCESSFULLY LOADED HTTPSHandler") + +from toolchain_util import (download_package, get_toolchain_compiler, Package, + package_directory) + LOG = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps") @@ -84,7 +100,10 @@ def create_virtualenv(): file.extract(member, build_dir) file.close() python_cmd = detect_python_cmd() - exec_cmd([python_cmd, find_file(build_dir, "virtualenv*", "virtualenv.py"), "--quiet", + output = exec_cmd([python_cmd, os.path.join(os.path.dirname(__file__), "tmp.py")]) + LOG.info("Ran tmp.py OK") + LOG.info(output) + exec_cmd([python_cmd, find_file(build_dir, "virtualenv*", "virtualenv.py"), "--python", python_cmd, ENV_DIR]) shutil.rmtree(build_dir) @@ -190,20 +209,32 @@ def find_file(*paths): def detect_python_cmd(): - '''Returns the system command that provides python 2.6 or greater.''' + '''Returns the a Python command that provides python 2.7 or greater. If an appropriate + system python is not available, download Python from the toolchain.''' paths = os.getenv("PATH").split(os.path.pathsep) - for cmd in ("python", "python27", "python2.7", "python-27", "python-2.7", "python26", - "python2.6", "python-26", "python-2.6"): + for cmd in ("python", "python27", "python2.7", "python-27", "python-2.7"): for path in paths: cmd_path = os.path.join(path, cmd) if not os.path.exists(cmd_path) or not os.access(cmd_path, os.X_OK): continue exit = subprocess.call([cmd_path, "-c", textwrap.dedent(""" import sys - sys.exit(int(sys.version_info[:2] < (2, 6)))""")]) + sys.exit(int(sys.version_info[:2] < (2, 7)))""")]) if exit == 0: return cmd_path - raise Exception("Could not find minimum required python version 2.6") + LOG.info("Could not find system python with version >= 2.7") + toolchain_root = os.environ.get("IMPALA_TOOLCHAIN") + if not toolchain_root: + raise Exception( + "Impala environment not set up correctly, make sure $IMPALA_TOOLCHAIN is set.") + pkg = Package("python") + download_package(toolchain_root, pkg, get_toolchain_compiler()) + python_cmd = os.path.join( + package_directory(toolchain_root, pkg.name, pkg.version), "bin/python") + if not os.path.exists(python_cmd): + raise Exception("Unexpected error bootstrapping python from toolchain: {0} does not " + "exist".format(python_cmd)) + return python_cmd def install_deps(): diff --git a/infra/python/lib/__init__.py b/infra/python/lib/__init__.py new file mode 100644 index 0000000..946a474 --- /dev/null +++ b/infra/python/lib/__init__.py @@ -0,0 +1 @@ +# This file is needed to make the files in this directory a python module diff --git a/infra/python/lib/toolchain_util.py b/infra/python/lib/toolchain_util.py new file mode 100644 index 0000000..b6f5dec --- /dev/null +++ b/infra/python/lib/toolchain_util.py @@ -0,0 +1,197 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Utilities for interacting with the impala toolchain. +# This module must be kept Python 2.6-compatible so that it can be used by +# bootstrap_virtualenv.py to download Python 2.7 on CentOS6. + +import logging +import os +import random +import re +import shutil +import subprocess +import time + +from collections import namedtuple + + +TOOLCHAIN_HOST = "https://native-toolchain.s3.amazonaws.com/build" + +# Maps return values from 'lsb_release -irs' to the corresponding OS labels for both the +# toolchain and the CDH components. +OsMapping = namedtuple('OsMapping', ['lsb_release', 'toolchain', 'cdh']) +OS_MAPPING = [ + OsMapping("centos5", "ec2-package-centos-5", None), + OsMapping("centos6", "ec2-package-centos-6", "redhat6"), + OsMapping("centos7", "ec2-package-centos-7", "redhat7"), + OsMapping("redhatenterpriseserver5", "ec2-package-centos-5", None), + OsMapping("redhatenterpriseserver6", "ec2-package-centos-6", "redhat6"), + OsMapping("redhatenterpriseserver7", "ec2-package-centos-7", "redhat7"), + OsMapping("debian6", "ec2-package-debian-6", None), + OsMapping("debian7", "ec2-package-debian-7", None), + OsMapping("debian8", "ec2-package-debian-8", "debian8"), + OsMapping("suselinux11", "ec2-package-sles-11", None), + OsMapping("suselinux12", "ec2-package-sles-12", "sles12"), + OsMapping("suse12.2", "ec2-package-sles-12", "sles12"), + OsMapping("suse12.3", "ec2-package-sles-12", "sles12"), + OsMapping("ubuntu12.04", "ec2-package-ubuntu-12-04", None), + OsMapping("ubuntu14.04", "ec2-package-ubuntu-14-04", None), + OsMapping("ubuntu15.04", "ec2-package-ubuntu-14-04", None), + OsMapping("ubuntu15.10", "ec2-package-ubuntu-14-04", None), + OsMapping('ubuntu16.04', "ec2-package-ubuntu-16-04", "ubuntu1604"), + OsMapping('ubuntu18.04', "ec2-package-ubuntu-18-04", "ubuntu1804") +] + + +class Package(object): + """ + Represents a package to be downloaded. A version, if not specified + explicitly, is retrieved from the environment variable IMPALA__VERSION. + URLs are retrieved from IMPALA__URL, but are optional. + """ + def __init__(self, name, version=None, url=None): + self.name = name + self.version = version + self.url = url + package_env_name = name.replace("-", "_").upper() + if self.version is None: + version_env_var = "IMPALA_{0}_VERSION".format(package_env_name) + + self.version = os.environ.get(version_env_var) + if not self.version: + raise Exception("Could not find version for {0} in environment var {1}".format( + name, version_env_var)) + if self.url is None: + url_env_var = "IMPALA_{0}_URL".format(package_env_name) + self.url = os.environ.get(url_env_var) + + +def get_toolchain_compiler(): + """Return the - string for the compiler package to use for the + toolchain.""" + # Currently we always use GCC. + return "gcc-{0}".format(os.environ["IMPALA_GCC_VERSION"]) + + +def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber): + if not download_path.endswith("/" + file_name): + raise Exception("URL {0} does not match with expected file_name {1}" + .format(download_path, file_name)) + NUM_ATTEMPTS = 3 + for attempt in range(1, NUM_ATTEMPTS + 1): + logging.info("Downloading {0} to {1}/{2} (attempt {3})".format( + download_path, destination, file_name, attempt)) + # --no-clobber avoids downloading the file if a file with the name already exists + try: + cmd = ["wget", download_path, "--directory-prefix={0}".format(destination)] + if wget_no_clobber: + cmd.append("--no-clobber") + check_output(cmd) + break + except Exception, e: + if attempt == NUM_ATTEMPTS: + raise + logging.error("Download failed; retrying after sleep: " + str(e)) + time.sleep(10 + random.random() * 5) # Sleep between 10 and 15 seconds. + logging.info("Extracting {0}".format(file_name)) + check_output(["tar", "xzf", os.path.join(destination, file_name), + "--directory={0}".format(destination)]) + os.unlink(os.path.join(destination, file_name)) + + +def download_package(destination, package, compiler, platform_release=None): + remove_existing_package(destination, package.name, package.version) + + toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID"] + label = get_platform_release_label(release=platform_release).toolchain + format_params = {'product': package.name, 'version': package.version, + 'compiler': compiler, 'label': label, 'toolchain_build_id': toolchain_build_id} + file_name = "{product}-{version}-{compiler}-{label}.tar.gz".format(**format_params) + format_params['file_name'] = file_name + if package.url is None: + url_path = "/{toolchain_build_id}/{product}/{version}-{compiler}/{file_name}".format( + **format_params) + download_path = TOOLCHAIN_HOST + url_path + else: + download_path = package.url + + wget_and_unpack_package(download_path, file_name, destination, True) + + +def remove_existing_package(toolchain_root, pkg_name, pkg_version): + dir_path = package_directory(toolchain_root, pkg_name, pkg_version) + if os.path.exists(dir_path): + logging.info("Removing existing package directory {0}".format(dir_path)) + shutil.rmtree(dir_path) + + +def package_directory(toolchain_root, pkg_name, pkg_version): + dir_name = "{0}-{1}".format(pkg_name, pkg_version) + return os.path.join(toolchain_root, dir_name) + + +def try_get_platform_release_label(): + """Gets the right package label from the OS version. Returns an OsMapping with both + 'toolchain' and 'cdh' labels. Return None if not found. + """ + try: + return get_platform_release_label() + except Exception, e: + logging.info("Could not get platform release label:" + str(e)) + return None + + +# Cache "lsb_release -irs" to avoid excessive logging from sh, and +# to shave a little bit of time. +lsb_release_cache = None + + +def get_platform_release_label(release=None): + """Gets the right package label from the OS version. Raise exception if not found. + 'release' can be provided to override the underlying OS version. + """ + global lsb_release_cache + if not release: + if lsb_release_cache: + release = lsb_release_cache + else: + lsb_release = check_output(["lsb_release", "-irs"]) + release = "".join(map(lambda x: x.lower(), lsb_release.split())) + # Only need to check against the major release if RHEL or CentOS + for platform in ['centos', 'redhatenterpriseserver']: + if platform in release: + release = release.split('.')[0] + break + lsb_release_cache = release + for mapping in OS_MAPPING: + if re.search(mapping.lsb_release, release): + return mapping + raise Exception("Could not find package label for OS version: {0}.".format(release)) + + +def check_output(cmd_args): + """Run the command and return the output. Raise an exception if the command returns + a non-zero return code. Similar to subprocess.check_output() which is only provided + in python 2.7. + """ + process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, _ = process.communicate() + if process.wait() != 0: + raise Exception("Command with args '%s' failed with exit code %s:\n%s" + % (cmd_args, process.returncode, stdout)) + return stdout diff --git a/infra/python/tmp.py b/infra/python/tmp.py new file mode 100644 index 0000000..365d400 --- /dev/null +++ b/infra/python/tmp.py @@ -0,0 +1,17 @@ +from __future__ import print_function +import sys +import os + +print("PATH IS: " + str(sys.path)) +try: + print("PYTHONPATH " + os.environ.get("PYTHONPATH")) +except: + pass + +# Ensure the ssl modules are loaded correctly. +import ssl +from httplib import HTTPConnection +from urllib2 import HTTPSHandler + + +print("SUCCESSFULLY LOADED HTTPSHandler") diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index 0be8c95..b88185b 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -29,6 +29,7 @@ import requests import shutil import socket import subprocess +import sys import tempfile import time from functools import wraps @@ -141,6 +142,7 @@ class ImpalaTestSuite(BaseTestSuite): @classmethod def setup_class(cls): """Setup section that runs before each test suite""" + assert sys.version_info > (2, 7), "Tests only support Python 2.7+" cls.hive_client, cls.client, cls.hs2_client = [None, None, None] # Create a Hive Metastore Client (used for executing some test SETUP steps metastore_host, metastore_port = pytest.config.option.metastore_server.split(':') -- 2.7.4