From 54d0eaa2df0dff796d01555ba0bd6527d07649db Mon Sep 17 00:00:00 2001 From: Tim Armstrong Date: Thu, 27 Dec 2018 08:44:40 -0800 Subject: [PATCH] Switch to gcc7 TODO: * Turn on unused Status warning * Investigate if snappy got slower - try building with -O3? * CPU time comparison? Testing: TODO: * Run exhaustive tests * Run ASAN tests * Run exhaustive release tests * Local perf run Report Generated on 2019-01-10 Run Description: "7cc909221241b6fea7096465cbb01b1e2dcb1a0a vs 24b309869df041086dbca147e51af12ffd170dd3" Cluster Name: UNKNOWN Lab Run Info: UNKNOWN Impala Version: impalad version 3.2.0-SNAPSHOT RELEASE () Baseline Impala Version: impalad version 3.2.0-SNAPSHOT RELEASE (2019-01-04) +----------+-----------------------+---------+------------+------------+----------------+ | Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) | +----------+-----------------------+---------+------------+------------+----------------+ | TPCH(10) | parquet / none / none | 4.37 | +2.65% | 2.31 | +0.60% | +----------+-----------------------+---------+------------+------------+----------------+ +----------+-----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+ | Workload | Query | File Format | Avg(s) | Base Avg(s) | Delta(Avg) | StdDev(%) | Base StdDev(%) | Iters | Median Diff(%) | MW Zval | Tval | +----------+-----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+ | TPCH(10) | TPCH-AGG1 | parquet / none / none | 19.78 | 16.93 | +16.84% | * 26.31% * | 3.83% | 20 | +2.97% | 1.59 | 2.43 | | TPCH(10) | TPCH-Q13 | parquet / none / none | 4.12 | 4.06 | +1.37% | 2.88% | 4.23% | 20 | +1.87% | 1.82 | 1.19 | | TPCH(10) | TPCH-Q21 | parquet / none / none | 8.93 | 8.83 | +1.10% | 1.29% | 0.73% | 20 | +1.16% | 2.82 | 3.29 | | TPCH(10) | TPCH-Q14 | parquet / none / none | 1.21 | 1.19 | +1.55% | 2.50% | 2.54% | 20 | +0.22% | 1.56 | 1.93 | | TPCH(10) | TPCH-Q11 | parquet / none / none | 0.77 | 0.77 | +0.89% | 2.07% | 3.76% | 20 | +0.04% | 0.31 | 0.93 | | TPCH(10) | TPCH-Q8 | parquet / none / none | 2.37 | 2.35 | +0.70% | 2.81% | 2.57% | 20 | +0.13% | 0.95 | 0.82 | | TPCH(10) | TPCH-Q20 | parquet / none / none | 1.24 | 1.23 | +0.78% | 3.00% | 2.36% | 20 | +0.03% | 0.48 | 0.91 | | TPCH(10) | TPCH-Q5 | parquet / none / none | 2.16 | 2.14 | +0.70% | 1.37% | 1.97% | 20 | +0.11% | 1.01 | 1.30 | | TPCH(10) | TPCH-Q12 | parquet / none / none | 1.11 | 1.10 | +0.50% | 3.74% | 2.91% | 20 | +0.00% | 0.07 | 0.47 | | TPCH(10) | TPCH-Q17 | parquet / none / none | 1.40 | 1.39 | +0.56% | 6.24% | 2.66% | 20 | -0.07% | -0.51 | 0.37 | | TPCH(10) | TPCH-Q1 | parquet / none / none | 3.98 | 3.96 | +0.40% | 1.13% | 0.96% | 20 | +0.03% | 0.92 | 1.20 | | TPCH(10) | TPCH-Q10 | parquet / none / none | 2.06 | 2.05 | +0.26% | 2.44% | 2.54% | 20 | +0.05% | 0.36 | 0.33 | | TPCH(10) | TPCH-Q19 | parquet / none / none | 1.68 | 1.67 | +0.28% | 1.97% | 1.83% | 20 | +0.02% | 0.28 | 0.46 | | TPCH(10) | TPCH-Q4 | parquet / none / none | 0.89 | 0.89 | +0.17% | 3.26% | 3.34% | 20 | -0.04% | -0.54 | 0.17 | | TPCH(10) | TPCH-AGG2 | parquet / none / none | 30.33 | 30.41 | -0.25% | 2.25% | 3.56% | 20 | +0.33% | 0.39 | -0.27 | | TPCH(10) | TPCH-Q16 | parquet / none / none | 1.16 | 1.16 | -0.01% | 1.80% | 3.01% | 20 | -0.03% | -0.16 | -0.01 | | TPCH(10) | TPCH-Q6 | parquet / none / none | 0.55 | 0.55 | -0.11% | 3.96% | 4.79% | 20 | +0.04% | 0.31 | -0.08 | | TPCH(10) | TPCH-Q22 | parquet / none / none | 1.17 | 1.18 | -0.09% | 1.59% | 1.57% | 20 | -0.05% | -0.89 | -0.18 | | TPCH(10) | TPCH-Q18 | parquet / none / none | 5.81 | 5.82 | -0.26% | 1.30% | 1.40% | 20 | -0.01% | -0.25 | -0.60 | | TPCH(10) | TPCH-AGG3 | parquet / none / none | 1.26 | 1.27 | -0.48% | 2.03% | 0.90% | 20 | -0.02% | -0.51 | -0.98 | | TPCH(10) | TPCH-Q15 | parquet / none / none | 1.44 | 1.44 | -0.48% | 3.05% | 2.49% | 20 | -0.18% | -0.89 | -0.55 | | TPCH(10) | TPCH-Q9 | parquet / none / none | 5.65 | 5.67 | -0.22% | 3.15% | 1.50% | 20 | -0.80% | -0.60 | -0.28 | | TPCH(10) | TPCH-Q3 | parquet / none / none | 1.86 | 1.88 | -1.07% | 1.71% | 1.71% | 20 | -0.25% | -1.56 | -2.00 | | TPCH(10) | TPCH-Q2 | parquet / none / none | 0.87 | 0.90 | -2.34% | 4.34% | 3.31% | 20 | -0.24% | -1.15 | -1.95 | | TPCH(10) | TPCH-Q7 | parquet / none / none | 7.40 | 7.53 | -1.72% | 2.76% | 2.99% | 20 | -1.99% | -2.12 | -1.91 | +----------+-----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+ (V) Significant Variability: TPCH(10) TPCH-AGG1 [parquet / none / none] (3.83% -> 26.31%) +--------------+------------+-----------+----------------+------------------+--------+---------+-----------+ | Operator | % of Query | StdDev(%) | Base StdDev(%) | Delta(StdDev(%)) | #Hosts | #Rows | Est #Rows | +--------------+------------+-----------+----------------+------------------+--------+---------+-----------+ | 02:HASH JOIN | 36.07% | 66.33% | 10.59% | +526.35% | 1 | 149.89M | 24.15M | +--------------+------------+-----------+----------------+------------------+--------+---------+-----------+ Change-Id: Iaed946401785749ed114c56baef357a5e56deae5 --- CMakeLists.txt | 7 ++++--- be/CMakeLists.txt | 7 ++++++- be/src/exprs/slot-ref.cc | 8 ++++++-- be/src/runtime/buffered-tuple-stream-test.cc | 4 +++- be/src/runtime/bufferpool/buffer-pool-test.cc | 4 +++- be/src/runtime/io/disk-io-mgr-test.cc | 4 +++- be/src/testutil/cpu-util.h | 4 +++- bin/bootstrap_toolchain.py | 15 +++++++++++---- bin/distcc/README.md | 1 + bin/impala-config.sh | 27 +++++++++++++++------------ infra/python/bootstrap_virtualenv.py | 5 ++++- testdata/datasets/tpcds/preload | 2 +- testdata/datasets/tpch/preload | 2 +- 13 files changed, 61 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fcc855..6f2325f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,20 +83,21 @@ function(set_dep_root NAME) string(TOLOWER ${NAME} NAME_LOWER) string(REPLACE "_" "-" NAME_LOWER ${NAME_LOWER}) set(VAL_NAME "IMPALA_${NAME}_VERSION") - set(${NAME}_ROOT $ENV{IMPALA_TOOLCHAIN}/${NAME_LOWER}-$ENV{${VAL_NAME}} PARENT_SCOPE) + set(${NAME}_ROOT $ENV{IMPALA_TOOLCHAIN_PACKAGE_ROOT}/${NAME_LOWER}-$ENV{${VAL_NAME}} PARENT_SCOPE) endfunction() +set(GCC_ROOT $ENV{IMPALA_TOOLCHAIN}/gcc-$ENV{IMPALA_GCC_VERSION}) # Define root path for all dependencies, this is in the form of # set_dep_root(PACKAGE) -> # PACKAGE_ROOT set to $ENV{IMPALA_TOOLCHAIN}/PACKAGE-$ENV{IMPALA_PACKAGE_VERSION} set_dep_root(AVRO) set_dep_root(ORC) +set_dep_root(BINUTILS) set_dep_root(BOOST) set_dep_root(BREAKPAD) set_dep_root(BZIP2) set_dep_root(CRCUTIL) set_dep_root(FLATBUFFERS) -set_dep_root(GCC) set_dep_root(GFLAGS) set_dep_root(GLOG) set_dep_root(GPERFTOOLS) @@ -104,7 +105,7 @@ set_dep_root(GTEST) set_dep_root(LIBEV) set_dep_root(LIBUNWIND) set_dep_root(LLVM) -set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION}) +set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN_PACKAGE_ROOT}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION}) set_dep_root(LZ4) set_dep_root(OPENLDAP) set_dep_root(PROTOBUF) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 2cec3dd..6cd2d2c 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -39,6 +39,8 @@ PROJECT(ASSEMBLER) # -Wno-vla: we use C99-style variable-length arrays # -pthread: enable multithreaded malloc # -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost +# -D_GLIBCXX_USE_CXX11_ABI=0: use the old ABI (the default one before gcc5) until we +# can switch all dependencies to the new ABI. # -fno-omit-frame-pointers: Keep frame pointer for functions in register SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer") @@ -46,7 +48,8 @@ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=c++14") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") -SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B $ENV{IMPALA_TOOLCHAIN}/binutils-$ENV{IMPALA_BINUTILS_VERSION}/bin/") +SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B ${BINUTILS_ROOT}/bin/") IF($ENV{USE_GOLD_LINKER} STREQUAL "true") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fuse-ld=gold") ENDIF() @@ -77,6 +80,7 @@ SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-zero-as-null-pointer-constant") SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-c++17-extensions") SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-inconsistent-missing-destructor-override") SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-return-type-c-linkage") +SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") # For any gcc builds: # -g: Enable symbols for profiler tools # -Wno-unused-local-typedefs: Do not warn for local typedefs that are unused. @@ -228,6 +232,7 @@ add_definitions(-DKUDU_HEADERS_USE_SHORT_STATUS_MACROS -DKUDU_HEADERS_USE_RICH_S # -DBOOST_NO_EXCEPTIONS: call a custom error handler for exceptions in codegen'd code. set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-std=c++14" "-DIR_COMPILE" "-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG" "-DBOOST_NO_EXCEPTIONS" + "-D_GLIBCXX_USE_CXX11_ABI=0" "-fcolor-diagnostics" "-Wno-deprecated" "-Wno-return-type-c-linkage" "-O1") # -Werror: compile warnings should be errors when using the toolchain compiler. set(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-Werror") diff --git a/be/src/exprs/slot-ref.cc b/be/src/exprs/slot-ref.cc index 00616b3..17fb9b7 100644 --- a/be/src/exprs/slot-ref.cc +++ b/be/src/exprs/slot-ref.cc @@ -467,8 +467,12 @@ DecimalVal SlotRef::GetDecimalVal( return DecimalVal(*reinterpret_cast(t->GetSlot(slot_offset_))); case 8: return DecimalVal(*reinterpret_cast(t->GetSlot(slot_offset_))); - case 16: - return DecimalVal(*reinterpret_cast(t->GetSlot(slot_offset_))); + case 16: { + // Use memcpy() to avoid unaligned load. + int128_t val; + memcpy(&val, t->GetSlot(slot_offset_), 16); + return DecimalVal(val); + } default: DCHECK(false); return DecimalVal::null(); diff --git a/be/src/runtime/buffered-tuple-stream-test.cc b/be/src/runtime/buffered-tuple-stream-test.cc index 6ff9805..e2a4aca 100644 --- a/be/src/runtime/buffered-tuple-stream-test.cc +++ b/be/src/runtime/buffered-tuple-stream-test.cc @@ -837,7 +837,9 @@ void SimpleTupleStreamTest::TestAttachMemory(bool pin_stream, bool attach_on_rea } else { EXPECT_EQ(0, num_buffers_attached) << "No buffers attached during iteration."; } - if (attach_on_read || !pin_stream) EXPECT_EQ(4, num_flushes); + if (attach_on_read || !pin_stream) { + EXPECT_EQ(4, num_flushes); + } out_batch->Reset(); stream.Close(out_batch, RowBatch::FlushMode::FLUSH_RESOURCES); if (attach_on_read) { diff --git a/be/src/runtime/bufferpool/buffer-pool-test.cc b/be/src/runtime/bufferpool/buffer-pool-test.cc index 87723d9..c663ce4 100644 --- a/be/src/runtime/bufferpool/buffer-pool-test.cc +++ b/be/src/runtime/bufferpool/buffer-pool-test.cc @@ -576,7 +576,9 @@ void BufferPoolTest::TestBufferAllocation(bool reserved) { BufferPool::ClientHandle client; ASSERT_OK(pool.RegisterClient("test client", NULL, &global_reservations_, NULL, TOTAL_MEM, NewProfile(), &client)); - if (reserved) ASSERT_TRUE(client.IncreaseReservationToFit(TOTAL_MEM)); + if (reserved) { + ASSERT_TRUE(client.IncreaseReservationToFit(TOTAL_MEM)); + } vector handles(NUM_BUFFERS); diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc index 6afdb5d..aaeb055 100644 --- a/be/src/runtime/io/disk-io-mgr-test.cc +++ b/be/src/runtime/io/disk-io-mgr-test.cc @@ -887,7 +887,9 @@ TEST_F(DiskIoMgrTest, MemScarcity) { data_offset += buffer->len(); buffers.emplace_back(move(buffer)); } - if (status.ok()) ASSERT_EQ(DATA_BYTES, data_offset); + if (status.ok()) { + ASSERT_EQ(DATA_BYTES, data_offset); + } for (auto& buffer : buffers) range->ReturnBuffer(move(buffer)); })); // Let the thread start running before starting the next. diff --git a/be/src/testutil/cpu-util.h b/be/src/testutil/cpu-util.h index a995a8e..885e3ca 100644 --- a/be/src/testutil/cpu-util.h +++ b/be/src/testutil/cpu-util.h @@ -38,7 +38,9 @@ class CpuTestUtil { CPU_SET(core, &cpuset); ASSERT_EQ(0, pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset)) << core; - if (validate_current_cpu) ASSERT_EQ(core, CpuInfo::GetCurrentCore()); + if (validate_current_cpu) { + ASSERT_EQ(core, CpuInfo::GetCurrentCore()); + } } /// Reset the thread affinity of the current thread to all cores. diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index d7bf5b4..4116901 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -190,13 +190,20 @@ def bootstrap(toolchain_root, packages): compiler = "gcc-{0}".format(os.environ["IMPALA_GCC_VERSION"]) def handle_package(p): - if check_for_existing_package(toolchain_root, p.name, p.version, compiler): + # Put packages compiled with different compilers into different subdirectories. + # We don't need to put GCC itself in a subdirectory. + if p.name == "gcc": + subdir = toolchain_root + else: + subdir = os.path.join(toolchain_root, "packages-" + compiler + "-oldabi") + + if check_for_existing_package(subdir, p.name, p.version, compiler): return if p.name != "kudu" or os.environ["KUDU_IS_SUPPORTED"] == "true": - download_package(toolchain_root, p, compiler) + download_package(subdir, p, compiler) else: - build_kudu_stub(toolchain_root, p.version, compiler) - write_version_file(toolchain_root, p.name, p.version, compiler, + build_kudu_stub(subdir, p.version, compiler) + write_version_file(subdir, p.name, p.version, compiler, get_platform_release_label().toolchain) execute_many(handle_package, packages) diff --git a/bin/distcc/README.md b/bin/distcc/README.md index a8487b2..809b4c6 100644 --- a/bin/distcc/README.md +++ b/bin/distcc/README.md @@ -25,6 +25,7 @@ The rest of the setup is done for you; here is a short description of what they # Usage +TODO: update usage for gcc7 ### First time 1. Symlink /opt/Impala-Toolchain to a directory writable by your user and point IMPALA_TOOLCHAIN at that directory. This ensures toolchain binaries are at the diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 9e27f54..c05e25e 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -68,12 +68,12 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=235-57748561c0 +export IMPALA_TOOLCHAIN_BUILD_ID=242-1989b83e29 # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p4 unset IMPALA_AVRO_URL -export IMPALA_BINUTILS_VERSION=2.26.1 +export IMPALA_BINUTILS_VERSION=2.28 unset IMPALA_BINUTILS_URL export IMPALA_BOOST_VERSION=1.57.0-p3 unset IMPALA_BOOST_URL @@ -91,7 +91,7 @@ export IMPALA_CYRUS_SASL_VERSION=2.1.23 unset IMPALA_CYRUS_SASL_URL export IMPALA_FLATBUFFERS_VERSION=1.6.0 unset IMPALA_FLATBUFFERS_URL -export IMPALA_GCC_VERSION=4.9.2 +export IMPALA_GCC_VERSION=7.4.0 unset IMPALA_GCC_URL export IMPALA_GDB_VERSION=7.9.1 unset IMPALA_GDB_URL @@ -122,7 +122,7 @@ export IMPALA_OPENLDAP_VERSION=2.4.25 unset IMPALA_OPENLDAP_URL export IMPALA_OPENSSL_VERSION=1.0.2l unset IMPALA_OPENSSL_URL -export IMPALA_ORC_VERSION=1.4.3-p2 +export IMPALA_ORC_VERSION=1.4.3-p3 unset IMPALA_ORC_URL export IMPALA_PROTOBUF_VERSION=3.5.1 unset IMPALA_PROTOBUF_URL @@ -442,6 +442,7 @@ export KUDU_MASTER_HOSTS="${KUDU_MASTER_HOSTS:-127.0.0.1}" export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}" export KUDU_MASTER_WEBUI_PORT="${KUDU_MASTER_WEBUI_PORT:-8051}" +export IMPALA_TOOLCHAIN_PACKAGE_ROOT="$IMPALA_TOOLCHAIN/packages-gcc-${IMPALA_GCC_VERSION}-oldabi" export IMPALA_FE_DIR="$IMPALA_HOME/fe" export IMPALA_BE_DIR="$IMPALA_HOME/be" export IMPALA_WORKLOAD_DIR="$IMPALA_HOME/testdata/workloads" @@ -449,8 +450,8 @@ export IMPALA_AUX_WORKLOAD_DIR="$IMPALA_AUX_TEST_HOME/testdata/workloads" export IMPALA_DATASET_DIR="$IMPALA_HOME/testdata/datasets" export IMPALA_AUX_DATASET_DIR="$IMPALA_AUX_TEST_HOME/testdata/datasets" export IMPALA_COMMON_DIR="$IMPALA_HOME/common" -export PATH="$IMPALA_TOOLCHAIN/gdb-$IMPALA_GDB_VERSION/bin:$PATH" -export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH" +export PATH="$IMPALA_TOOLCHAIN_PACKAGE_ROOT/gdb-$IMPALA_GDB_VERSION/bin:$PATH" +export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN_PACKAGE_ROOT/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH" # The directory in which all the thirdparty CDH components live. if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then @@ -603,14 +604,14 @@ if $USE_CDH_KUDU; then export IMPALA_KUDU_HOME=${CDH_COMPONENTS_HOME}/kudu-$IMPALA_KUDU_VERSION else export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"4ec2598"} - export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN}/kudu-$IMPALA_KUDU_VERSION + export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/kudu-$IMPALA_KUDU_VERSION fi # Set $THRIFT_HOME to the Thrift directory in toolchain. -export THRIFT_HOME="${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}" +export THRIFT_HOME="${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/thrift-${IMPALA_THRIFT_VERSION}" # ASAN needs a matching version of llvm-symbolizer to symbolize stack traces. -export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer" +export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer" export CLUSTER_DIR="${IMPALA_HOME}/testdata/cluster" @@ -641,15 +642,17 @@ LIBHDFS_OPTS+=" -XX:ErrorFile=${IMPALA_LOGS_DIR}/hs_err_pid%p.log" # running tests that load UDF jars. LIBHDFS_OPTS="${LIBHDFS_OPTS} -XX:MaxPermSize=128m" -export IMPALA_SNAPPY_PATH="${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib" +export IMPALA_SNAPPY_PATH="${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/snappy-${IMPALA_SNAPPY_VERSION}/lib" export JAVA_LIBRARY_PATH="${IMPALA_SNAPPY_PATH}" -# So that the frontend tests and PlanService can pick up required libraries +# Set LD_LIBRARY_PATH to include libraries that may be loaded by binaries or tests. +# Override any user LD_LIBRARY_PATH. +# TODO: reduce reliance on LD_LIBRARY_PATH LIB_JAVA=`find "${JAVA_HOME}/" -name libjava.so | head -1` LIB_JSIG=`find "${JAVA_HOME}/" -name libjsig.so | head -1` LIB_JVM=` find "${JAVA_HOME}/" -name libjvm.so | head -1` -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" +export LD_LIBRARY_PATH="`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HADOOP_LIB_DIR}/native" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}" diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 305e588..e489f28 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -239,7 +239,6 @@ def install_compiled_deps_if_possible(): # problem with inline declarations in older libc headers. Setting -fgnu89-inline is a # workaround. distro_version = ''.join(exec_cmd(["lsb_release", "-irs"]).lower().split()) - print distro_version if distro_version.startswith("centos5."): env["CFLAGS"] = "-fgnu89-inline" @@ -295,6 +294,10 @@ def install_kudu_client_if_possible(): env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include") env["LIBRARY_PATH"] = os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'), os.path.join(kudu_client_dir, 'lib64')]) + # Compiled C++ code must use the same ABI as libkudu_client.so. Distutils doesn't + # distinguish C and C++ compiler options, so we need to pass in CFLAGS, not CXXFLAGS. + env["CFLAGS"] = "-D_GLIBCXX_USE_CXX11_ABI=0" + exec_pip_install(["-r", KUDU_REQS_PATH], cc=cc, env=env) mark_reqs_installed(KUDU_REQS_PATH) finally: diff --git a/testdata/datasets/tpcds/preload b/testdata/datasets/tpcds/preload index 423b3c8..9da1b63 100755 --- a/testdata/datasets/tpcds/preload +++ b/testdata/datasets/tpcds/preload @@ -31,7 +31,7 @@ then fi TPC_DS_DIRNAME=tpc-ds-${IMPALA_TPC_DS_VERSION} -TPC_DS_HOME=${IMPALA_TOOLCHAIN}/${TPC_DS_DIRNAME} +TPC_DS_HOME=${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/${TPC_DS_DIRNAME} echo "Generating TPC-DS data into ${TPC_DS_DATA}" # Delete any preexisting data or symlinks diff --git a/testdata/datasets/tpch/preload b/testdata/datasets/tpch/preload index 619a4d2..128862c 100755 --- a/testdata/datasets/tpch/preload +++ b/testdata/datasets/tpch/preload @@ -30,7 +30,7 @@ then TPC_H_DATA=${TPC_H_DATA}${SCALE_FACTOR} fi -TPC_H_HOME=${IMPALA_TOOLCHAIN}/tpc-h-${IMPALA_TPC_H_VERSION} +TPC_H_HOME=${IMPALA_TOOLCHAIN_PACKAGE_ROOT}/tpc-h-${IMPALA_TPC_H_VERSION} TPC_H_DBGEN=${TPC_H_HOME}/bin/dbgen if [ ! -x ${TPC_H_DBGEN} ]; then -- 2.7.4