diff --git beeline/pom.xml beeline/pom.xml
index 4567d5e09b..19ec53eba6 100644
--- beeline/pom.xml
+++ beeline/pom.xml
@@ -55,6 +55,11 @@
hive-jdbc
${project.version}
+
+ org.apache.hive
+ hive-standalone-metastore-server
+ ${project.version}
+
commons-cli
diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 4eda8e3ff5..29ec2de68c 100644
--- beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -1026,12 +1026,15 @@ public void updateOptsForCli() {
getOpts().setNullEmptyString(true);
}
+ public int begin(String[] args, InputStream inputStream) throws IOException {
+ return begin(args, inputStream, true);
+ }
/**
* Start accepting input from stdin, and dispatch it
* to the appropriate {@link CommandHandler} until the
* global variable exit is true.
*/
- public int begin(String[] args, InputStream inputStream) throws IOException {
+ public int begin(String[] args, InputStream inputStream, boolean keepHistory) throws IOException {
try {
// load the options first, so we can override on the command line
getOpts().load();
@@ -1039,7 +1042,9 @@ public int begin(String[] args, InputStream inputStream) throws IOException {
// nothing
}
- setupHistory();
+ if (keepHistory) {
+ setupHistory();
+ }
//add shutdown hook to cleanup the beeline for smooth exit
addBeelineShutdownHook();
@@ -1341,7 +1346,11 @@ public ConsoleReader initializeConsoleReader(InputStream inputStream) throws IOE
try {
// now set the output for the history
- consoleReader.setHistory(this.history);
+ if (this.history != null) {
+ consoleReader.setHistory(this.history);
+ } else {
+ consoleReader.setHistoryEnabled(false);
+ }
} catch (Exception e) {
handleException(e);
}
diff --git beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
index 69514e51b7..c7234760e7 100644
--- beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
+++ beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
@@ -100,7 +100,7 @@ protected void execSql(String sqlScriptFile) throws IOException {
// we always add a line separator at the end while calling dbCommandParser.buildCommand.
beeLine.getOpts().setEntireLineAsCommand(true);
LOG.debug("Going to run command <" + builder.buildToLog() + ">");
- int status = beeLine.begin(builder.buildToRun(), null);
+ int status = beeLine.begin(builder.buildToRun(), null, false);
if (status != 0) {
throw new IOException("Schema script failed, errorcode " + status);
}
diff --git bin/hive bin/hive
index 1ade51eebd..a7ae2f571e 100755
--- bin/hive
+++ bin/hive
@@ -356,6 +356,7 @@ fi
# include the log4j jar that is used for hive into the classpath
CLASSPATH="${CLASSPATH}:${LOG_JAR_CLASSPATH}"
export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${LOG_JAR_CLASSPATH}"
+export JVM_PID="$$"
if [ "$TORUN" = "" ] ; then
echo "Service $SERVICE not found"
diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index aa58d7445c..3e38fe66c5 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1679,7 +1679,15 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal
"joins unnecessary memory will be allocated and then trimmed."),
HIVEHYBRIDGRACEHASHJOINBLOOMFILTER("hive.mapjoin.hybridgrace.bloomfilter", true, "Whether to " +
"use BloomFilter in Hybrid grace hash join to minimize unnecessary spilling."),
-
+ HIVEMAPJOINFULLOUTER("hive.mapjoin.full.outer", true,
+ "Whether to use MapJoin for FULL OUTER JOINs."),
+ HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE(
+ "hive.test.mapjoin.full.outer.override",
+ "none", new StringSet("none", "enable", "disable"),
+ "internal use only, used to override the hive.mapjoin.full.outer\n" +
+ "setting. Using enable will force it on and disable will force it off.\n" +
+ "The default none is do nothing, of course",
+ true),
HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", 10000,
"How many rows with the same key value should be cached in memory per smb joined table."),
HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000,
@@ -1701,6 +1709,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal
"If the bucketing/sorting properties of the table exactly match the grouping key, whether to perform \n" +
"the group by in the mapper by using BucketizedHiveInputFormat. The only downside to this\n" +
"is that it limits the number of mappers to the number of files."),
+ HIVE_DEFAULT_NULLS_LAST("hive.default.nulls.last", true,
+ "Whether to set NULLS LAST as the default null ordering"),
HIVE_GROUPBY_POSITION_ALIAS("hive.groupby.position.alias", false,
"Whether to enable using Column Position Alias in Group By"),
HIVE_ORDERBY_POSITION_ALIAS("hive.orderby.position.alias", true,
@@ -2035,7 +2045,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal
"However, if it is on, and the predicted size of the larger input for a given join is greater \n" +
"than this number, the join will not be converted to a dynamically partitioned hash join. \n" +
"The value \"-1\" means no limit."),
- HIVEHASHTABLEKEYCOUNTADJUSTMENT("hive.hashtable.key.count.adjustment", 2.0f,
+ HIVEHASHTABLEKEYCOUNTADJUSTMENT("hive.hashtable.key.count.adjustment", 0.99f,
"Adjustment to mapjoin hashtable size derived from table and column statistics; the estimate" +
" of the number of keys is divided by this value. If the value is 0, statistics are not used" +
"and hive.hashtable.initialCapacity is used instead."),
diff --git common/src/java/org/apache/hive/common/util/ProcessUtils.java common/src/java/org/apache/hive/common/util/ProcessUtils.java
new file mode 100644
index 0000000000..409384fc00
--- /dev/null
+++ common/src/java/org/apache/hive/common/util/ProcessUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Process related utilities.
+ */
+public class ProcessUtils {
+ private static Logger LOG = LoggerFactory.getLogger(ProcessUtils.class);
+
+ public static Integer getPid() {
+ // JVM_PID is exported by bin/hive
+ String pidStr = System.getenv("JVM_PID");
+
+ // in case if it is not set correctly used fallback from mxbean which is implementation specific
+ if (pidStr == null || pidStr.trim().isEmpty()) {
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ if (name != null) {
+ int idx = name.indexOf("@");
+ if (idx != -1) {
+ pidStr = name.substring(0, name.indexOf("@"));
+ }
+ }
+ }
+ try {
+ if (pidStr != null) {
+ return Integer.valueOf(pidStr);
+ }
+ } catch (NumberFormatException nfe) {
+ // ignore
+ }
+ return null;
+ }
+
+ public static Process runCmdAsync(List cmd) {
+ try {
+ LOG.info("Running command async: " + cmd);
+ return new ProcessBuilder(cmd).inheritIO().start();
+ } catch (IOException ex) {
+ throw new IllegalStateException(ex);
+ }
+ }
+}
diff --git common/src/java/org/apache/hive/http/HttpServer.java common/src/java/org/apache/hive/http/HttpServer.java
index 3cb7a33c4e..24c5422a18 100644
--- common/src/java/org/apache/hive/http/HttpServer.java
+++ common/src/java/org/apache/hive/http/HttpServer.java
@@ -21,6 +21,9 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -558,6 +561,22 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept
addServlet("conf", "/conf", ConfServlet.class);
addServlet("stacks", "/stacks", StackServlet.class);
addServlet("conflog", "/conflog", Log4j2ConfiguratorServlet.class);
+ final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome();
+ if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) {
+ addServlet("prof", "/prof", ProfileServlet.class);
+ Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR);
+ if (Files.notExists(tmpDir)) {
+ Files.createDirectories(tmpDir);
+ }
+ ServletContextHandler genCtx =
+ new ServletContextHandler(contexts, "/prof-output");
+ setContextAttributes(genCtx.getServletContext(), b.contextAttrs);
+ genCtx.addServlet(ProfileOutputServlet.class, "/*");
+ genCtx.setResourceBase(tmpDir.toAbsolutePath().toString());
+ genCtx.setDisplayName("prof-output");
+ } else {
+ LOG.info("ASYNC_PROFILER_HOME env or -Dasync.profiler.home not specified. Disabling /prof endpoint..");
+ }
for (Pair> p : b.servlets) {
addServlet(p.getFirst(), "/" + p.getFirst(), p.getSecond());
diff --git common/src/java/org/apache/hive/http/ProfileOutputServlet.java common/src/java/org/apache/hive/http/ProfileOutputServlet.java
new file mode 100644
index 0000000000..fdca1f3cb3
--- /dev/null
+++ common/src/java/org/apache/hive/http/ProfileOutputServlet.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.http;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Servlet to serve files generated by {@link ProfileServlet}
+ */
+public class ProfileOutputServlet extends DefaultServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileOutputServlet.class);
+
+ @Override
+ protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
+ throws ServletException, IOException {
+ String absoluteDiskPath = getServletContext().getRealPath(req.getPathInfo());
+ File requestedFile = new File(absoluteDiskPath);
+ // async-profiler version 1.4 writes 'Started [cpu] profiling' to output file when profiler is running which
+ // gets replaced by final output. If final output is not ready yet, the file size will be <100 bytes (in all modes).
+ if (requestedFile.length() < 100) {
+ LOG.info("{} is incomplete. Sending auto-refresh header..", requestedFile);
+ resp.setHeader("Refresh", "2," + req.getRequestURI());
+ resp.getWriter().write("This page will auto-refresh every 2 second until output file is ready..");
+ } else {
+ super.doGet(req, resp);
+ }
+ }
+}
\ No newline at end of file
diff --git common/src/java/org/apache/hive/http/ProfileServlet.java common/src/java/org/apache/hive/http/ProfileServlet.java
new file mode 100644
index 0000000000..48437563b4
--- /dev/null
+++ common/src/java/org/apache/hive/http/ProfileServlet.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.http;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hive.common.util.ProcessUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+
+/**
+ * Servlet that runs async-profiler as web-endpoint.
+ * Following options from async-profiler can be specified as query paramater.
+ * // -e event profiling event: cpu|alloc|lock|cache-misses etc.
+ * // -d duration run profiling for seconds (integer)
+ * // -i interval sampling interval in nanoseconds (long)
+ * // -j jstackdepth maximum Java stack depth (integer)
+ * // -b bufsize frame buffer size (long)
+ * // -t profile different threads separately
+ * // -s simple class names instead of FQN
+ * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr
+ * // --width px SVG width pixels (integer)
+ * // --height px SVG frame height pixels (integer)
+ * // --minwidth px skip frames smaller than px (double)
+ * // --reverse generate stack-reversed FlameGraph / Call tree
+ * Example:
+ * - To collect 30 second CPU profile of current process (returns FlameGraph svg)
+ * curl "http://localhost:10002/prof"
+ * - To collect 1 minute CPU profile of current process and output in tree format (html)
+ * curl "http://localhost:10002/prof?output=tree&duration=60"
+ * - To collect 30 second heap allocation profile of current process (returns FlameGraph svg)
+ * curl "http://localhost:10002/prof?event=alloc"
+ * - To collect lock contention profile of current process (returns FlameGraph svg)
+ * curl "http://localhost:10002/prof?event=lock"
+ * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events)
+ * // Perf events:
+ * // cpu
+ * // page-faults
+ * // context-switches
+ * // cycles
+ * // instructions
+ * // cache-references
+ * // cache-misses
+ * // branches
+ * // branch-misses
+ * // bus-cycles
+ * // L1-dcache-load-misses
+ * // LLC-load-misses
+ * // dTLB-load-misses
+ * // mem:breakpoint
+ * // trace:tracepoint
+ * // Java events:
+ * // alloc
+ * // lock
+ */
+public class ProfileServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
+ private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+ private static final String ALLOWED_METHODS = "GET";
+ private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+ private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
+ private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
+ private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
+ private static final String PROFILER_SCRIPT = "/profiler.sh";
+ private static final int DEFAULT_DURATION_SECONDS = 10;
+ private static final AtomicInteger ID_GEN = new AtomicInteger(0);
+ static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
+
+ enum Event {
+ CPU("cpu"),
+ ALLOC("alloc"),
+ LOCK("lock"),
+ PAGE_FAULTS("page-faults"),
+ CONTEXT_SWITCHES("context-switches"),
+ CYCLES("cycles"),
+ INSTRUCTIONS("instructions"),
+ CACHE_REFERENCES("cache-references"),
+ CACHE_MISSES("cache-misses"),
+ BRANCHES("branches"),
+ BRANCH_MISSES("branch-misses"),
+ BUS_CYCLES("bus-cycles"),
+ L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
+ LLC_LOAD_MISSES("LLC-load-misses"),
+ DTLB_LOAD_MISSES("dTLB-load-misses"),
+ MEM_BREAKPOINT("mem:breakpoint"),
+ TRACE_TRACEPOINT("trace:tracepoint"),;
+
+ private String internalName;
+
+ Event(final String internalName) {
+ this.internalName = internalName;
+ }
+
+ public String getInternalName() {
+ return internalName;
+ }
+
+ public static Event fromInternalName(final String name) {
+ for (Event event : values()) {
+ if (event.getInternalName().equalsIgnoreCase(name)) {
+ return event;
+ }
+ }
+
+ return null;
+ }
+ }
+
+ enum Output {
+ SUMMARY,
+ TRACES,
+ FLAT,
+ COLLAPSED,
+ SVG,
+ TREE,
+ JFR
+ }
+
+ private Lock profilerLock = new ReentrantLock();
+ private Integer pid;
+ private String asyncProfilerHome;
+ private Process process;
+
+ public ProfileServlet() {
+ this.asyncProfilerHome = getAsyncProfilerHome();
+ this.pid = ProcessUtils.getPid();
+ LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome);
+ }
+
+ @Override
+ protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws IOException {
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
+ resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ setResponseHeader(resp);
+ resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
+ return;
+ }
+
+ // make sure async profiler home is set
+ if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+ resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(resp);
+ resp.getWriter().write("ASYNC_PROFILER_HOME env is not set.");
+ return;
+ }
+
+ // if pid is explicitly specified, use it else default to current process
+ pid = getInteger(req, "pid", pid);
+
+ // if pid is not specified in query param and if current process pid cannot be determined
+ if (pid == null) {
+ resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(resp);
+ resp.getWriter().write("'pid' query parameter unspecified or unable to determine PID of current process.");
+ return;
+ }
+
+ final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
+ final Output output = getOutput(req);
+ final Event event = getEvent(req);
+ final Long interval = getLong(req, "interval");
+ final Integer jstackDepth = getInteger(req, "jstackdepth", null);
+ final Long bufsize = getLong(req, "bufsize");
+ final boolean thread = req.getParameterMap().containsKey("thread");
+ final boolean simple = req.getParameterMap().containsKey("simple");
+ final Integer width = getInteger(req, "width", null);
+ final Integer height = getInteger(req, "height", null);
+ final Double minwidth = getMinWidth(req);
+ final boolean reverse = req.getParameterMap().containsKey("reverse");
+
+ if (process == null || !process.isAlive()) {
+ try {
+ int lockTimeoutSecs = 3;
+ if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
+ try {
+ File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" +
+ event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." +
+ output.name().toLowerCase());
+ List cmd = new ArrayList<>();
+ cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
+ cmd.add("-e");
+ cmd.add(event.getInternalName());
+ cmd.add("-d");
+ cmd.add("" + duration);
+ cmd.add("-o");
+ cmd.add(output.name().toLowerCase());
+ cmd.add("-f");
+ cmd.add(outputFile.getAbsolutePath());
+ if (interval != null) {
+ cmd.add("-i");
+ cmd.add(interval.toString());
+ }
+ if (jstackDepth != null) {
+ cmd.add("-j");
+ cmd.add(jstackDepth.toString());
+ }
+ if (bufsize != null) {
+ cmd.add("-b");
+ cmd.add(bufsize.toString());
+ }
+ if (thread) {
+ cmd.add("-t");
+ }
+ if (simple) {
+ cmd.add("-s");
+ }
+ if (width != null) {
+ cmd.add("--width");
+ cmd.add(width.toString());
+ }
+ if (height != null) {
+ cmd.add("--height");
+ cmd.add(height.toString());
+ }
+ if (minwidth != null) {
+ cmd.add("--minwidth");
+ cmd.add(minwidth.toString());
+ }
+ if (reverse) {
+ cmd.add("--reverse");
+ }
+ cmd.add(pid.toString());
+ process = ProcessUtils.runCmdAsync(cmd);
+
+ // set response and set refresh header to output location
+ setResponseHeader(resp);
+ resp.setStatus(HttpServletResponse.SC_ACCEPTED);
+ String relativeUrl = "/prof-output/" + outputFile.getName();
+ resp.getWriter().write(
+ "Started [" + event.getInternalName() + "] profiling. This page will automatically redirect to " +
+ relativeUrl + " after " + duration + " seconds.\n\ncommand:\n" + Joiner.on(" ").join(cmd));
+
+ // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified via url param
+ int refreshDelay = getInteger(req, "refreshDelay", 0);
+
+ // instead of sending redirect, set auto-refresh so that browsers will refresh with redirected url
+ resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
+ resp.getWriter().flush();
+ } finally {
+ profilerLock.unlock();
+ }
+ } else {
+ setResponseHeader(resp);
+ resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ resp.getWriter().write("Unable to acquire lock. Another instance of profiler might be running.");
+ LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be running.",
+ lockTimeoutSecs);
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while acquiring profile lock.", e);
+ resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ }
+ } else {
+ setResponseHeader(resp);
+ resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ resp.getWriter().write("Another instance of profiler is already running.");
+ }
+ }
+
+ private Integer getInteger(final HttpServletRequest req, final String param, final Integer defaultValue) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Integer.valueOf(value);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+ return defaultValue;
+ }
+
+ private Long getLong(final HttpServletRequest req, final String param) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Long.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Double getMinWidth(final HttpServletRequest req) {
+ final String value = req.getParameter("minwidth");
+ if (value != null) {
+ try {
+ return Double.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Event getEvent(final HttpServletRequest req) {
+ final String eventArg = req.getParameter("event");
+ if (eventArg != null) {
+ Event event = Event.fromInternalName(eventArg);
+ return event == null ? Event.CPU : event;
+ }
+ return Event.CPU;
+ }
+
+ private Output getOutput(final HttpServletRequest req) {
+ final String outputArg = req.getParameter("output");
+ if (req.getParameter("output") != null) {
+ try {
+ return Output.valueOf(outputArg.trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return Output.SVG;
+ }
+ }
+ return Output.SVG;
+ }
+
+ private void setResponseHeader(final HttpServletResponse response) {
+ response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
+ response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+ response.setContentType(CONTENT_TYPE_TEXT);
+ }
+
+ static String getAsyncProfilerHome() {
+ String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
+ // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set
+ if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+ asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
+ }
+
+ return asyncProfilerHome;
+ }
+}
diff --git data/files/fullouter_long_big_1a.txt data/files/fullouter_long_big_1a.txt
new file mode 100644
index 0000000000..8cf831fb4b
--- /dev/null
+++ data/files/fullouter_long_big_1a.txt
@@ -0,0 +1,11 @@
+-5310365297525168078
+-6187919478609154811
+968819023021777205
+3313583664488247651
+-5206670856103795573
+\N
+-6187919478609154811
+1569543799237464101
+-6187919478609154811
+-8460550397108077433
+-6187919478609154811
diff --git data/files/fullouter_long_big_1a_nonull.txt data/files/fullouter_long_big_1a_nonull.txt
new file mode 100644
index 0000000000..b2325adefb
--- /dev/null
+++ data/files/fullouter_long_big_1a_nonull.txt
@@ -0,0 +1,10 @@
+1569543799237464101
+-6187919478609154811
+968819023021777205
+-8460550397108077433
+-6187919478609154811
+-5310365297525168078
+-6187919478609154811
+-5206670856103795573
+3313583664488247651
+-6187919478609154811
diff --git data/files/fullouter_long_big_1b.txt data/files/fullouter_long_big_1b.txt
new file mode 100644
index 0000000000..87c2b3c42b
--- /dev/null
+++ data/files/fullouter_long_big_1b.txt
@@ -0,0 +1,13 @@
+\N
+31713
+31713
+31713
+31713
+32030
+31713
+-25394
+31713
+31713
+31713
+31713
+31713
diff --git data/files/fullouter_long_big_1c.txt data/files/fullouter_long_big_1c.txt
new file mode 100644
index 0000000000..2d13c260c9
--- /dev/null
+++ data/files/fullouter_long_big_1c.txt
@@ -0,0 +1,11 @@
+1928928239,\N
+-1437463633,YYXPPCH
+-1437463633,TKTKGVGFW
+1725068083,MKSCCE
+1928928239,\N
+\N,ABBZ
+1928928239,AMKTIWQ
+-1437463633,JU
+1928928239,VAQHVRI
+-1437463633,SOWDWMS
+-1437463633,\N
diff --git data/files/fullouter_long_big_1d.txt data/files/fullouter_long_big_1d.txt
new file mode 100644
index 0000000000..4137f67c6e
--- /dev/null
+++ data/files/fullouter_long_big_1d.txt
@@ -0,0 +1,12 @@
+-702028721
+-702028721
+-1780951928
+-670834064
+-814597051
+\N
+-814597051
+-814597051
+-702028721
+-2038654700
+\N
+-814597051
diff --git data/files/fullouter_long_small_1a.txt data/files/fullouter_long_small_1a.txt
new file mode 100644
index 0000000000..45d582508b
--- /dev/null
+++ data/files/fullouter_long_small_1a.txt
@@ -0,0 +1,54 @@
+-1339636982994067311,2000-06-20
+-2575185053386712613,2105-01-21
+\N,2098-02-10
+-6784441713807772877,1845-02-16
+\N,2024-01-23
+-4224290881682877258,2185-07-08
+-614848861623872247,2101-05-25
+-2098090254092150988,2163-05-26
+434940853096155515,2275-02-08
+3873405809071478736,2034-06-09
+-2184423060953067642,1880-10-06
+7297177530102477725,1921-05-11
+7937120928560087303,2083-03-14
+\N,2242-02-08
+-2688622006344936758,2129-01-11
+214451696109242839,1977-01-04
+-4961171400048338491,2196-08-10
+4436884039838843341,2031-05-23
+2438535236662373438,1916-01-10
+6049335087268933751,2282-06-09
+8755921538765428593,1827-05-01
+5252407779338300447,2039-03-10
+-2184423060953067642,1853-07-06
+7297177530102477725,1926-04-12
+-2098090254092150988,1817-03-12
+-5754527700632192146,1958-07-15
+-614848861623872247,2112-11-09
+5246983111579595707,1817-07-01
+-2098090254092150988,2219-12-23
+-5706981533666803767,2151-06-09
+7297177530102477725,2125-08-26
+-7707546703881534780,2134-08-20
+214451696109242839,2179-04-18
+3845554233155411208,1805-11-10
+3905351789241845882,2045-12-05
+2438535236662373438,2026-06-23
+-2688622006344936758,1948-10-15
+6049335087268933751,2086-12-17
+-2575185053386712613,1809-07-12
+-327698348664467755,2222-10-15
+-4224290881682877258,1813-05-17
+3873405809071478736,2164-04-23
+-5706981533666803767,1800-09-20
+214451696109242839,1855-05-12
+2438535236662373438,1881-09-16
+5252407779338300447,2042-04-26
+-3655445881497026796,2108-08-16
+3905351789241845882,1866-07-28
+-6784441713807772877,2054-06-17
+5246983111579595707,2260-05-11
+-1339636982994067311,2008-12-03
+3873405809071478736,1918-11-20
+-4224290881682877258,2120-01-16
+3845554233155411208,2264-04-05
diff --git data/files/fullouter_long_small_1a_nonull.txt data/files/fullouter_long_small_1a_nonull.txt
new file mode 100644
index 0000000000..bf94d5a866
--- /dev/null
+++ data/files/fullouter_long_small_1a_nonull.txt
@@ -0,0 +1,51 @@
+5246983111579595707,1817-07-01
+4436884039838843341,2031-05-23
+-4224290881682877258,1813-05-17
+-4961171400048338491,2196-08-10
+-2575185053386712613,2105-01-21
+5252407779338300447,2042-04-26
+-614848861623872247,2101-05-25
+-2098090254092150988,2163-05-26
+2438535236662373438,1881-09-16
+214451696109242839,2179-04-18
+2438535236662373438,2026-06-23
+-2184423060953067642,1853-07-06
+3873405809071478736,2164-04-23
+214451696109242839,1855-05-12
+-6784441713807772877,1845-02-16
+-2688622006344936758,1948-10-15
+7297177530102477725,1921-05-11
+-2575185053386712613,1809-07-12
+3905351789241845882,2045-12-05
+3845554233155411208,1805-11-10
+-3655445881497026796,2108-08-16
+3905351789241845882,1866-07-28
+-1339636982994067311,2008-12-03
+7297177530102477725,2125-08-26
+7297177530102477725,1926-04-12
+-5706981533666803767,1800-09-20
+6049335087268933751,2282-06-09
+3845554233155411208,2264-04-05
+8755921538765428593,1827-05-01
+-1339636982994067311,2000-06-20
+-2098090254092150988,1817-03-12
+3873405809071478736,2034-06-09
+2438535236662373438,1916-01-10
+5246983111579595707,2260-05-11
+-5706981533666803767,2151-06-09
+-614848861623872247,2112-11-09
+-327698348664467755,2222-10-15
+-2184423060953067642,1880-10-06
+434940853096155515,2275-02-08
+-4224290881682877258,2120-01-16
+-5754527700632192146,1958-07-15
+-4224290881682877258,2185-07-08
+-2098090254092150988,2219-12-23
+-7707546703881534780,2134-08-20
+214451696109242839,1977-01-04
+-2688622006344936758,2129-01-11
+7937120928560087303,2083-03-14
+-6784441713807772877,2054-06-17
+3873405809071478736,1918-11-20
+6049335087268933751,2086-12-17
+5252407779338300447,2039-03-10
diff --git data/files/fullouter_long_small_1b.txt data/files/fullouter_long_small_1b.txt
new file mode 100644
index 0000000000..7d45fe4120
--- /dev/null
+++ data/files/fullouter_long_small_1b.txt
@@ -0,0 +1,72 @@
+2748,2298-06-20 21:01:24
+11232,2533-11-26 12:22:18
+\N,2124-05-07 15:01:19.021
+3198,2428-06-13 16:21:33.955
+-7624,2219-12-03 17:07:19
+24870,2752-12-26 12:32:23.03685163
+14865,2943-03-21 00:42:10.505
+-8624,2644-05-04 04:45:07.839
+-30059,2269-05-04 21:23:44.000339209
+14865,2079-10-06 16:54:35.117
+-8435,2834-12-06 16:38:18.901
+10553,2168-05-05 21:10:59.000152113
+-8624,2282-03-28 07:58:16
+-15361,2219-09-15 20:15:03.000169887
+-14172,1918-09-13 11:44:24.496926711
+26484,1919-03-04 07:32:37.519
+-14172,2355-01-14 23:23:34
+-24775,2920-08-06 15:58:28.261059449
+-23117,2037-01-05 21:52:30.685952759
+17125,2236-07-14 01:54:40.927230276
+21181,2253-03-12 11:55:48.332
+-7373,2662-10-28 12:07:02.000526564
+-8087,2550-06-26 23:57:42.588007617
+29407,2385-12-14 06:03:39.597
+21181,2434-02-20 00:46:29.633
+-14172,2809-06-07 02:10:58
+13598,2421-05-20 14:18:31.000264698
+2748,2759-02-13 18:04:36.000307355
+-22422,1949-03-13 00:07:53.075
+26484,2953-03-10 02:05:26.508953676
+4510,2777-03-24 03:44:28.000169723
+-24775,2035-03-26 08:11:23.375224153
+-30059,2713-10-13 09:28:49
+-20517,2774-06-23 12:04:06.5
+11232,2038-04-06 14:53:59
+32030,2101-09-09 07:35:05.145
+-29600,2333-11-02 15:06:30
+-30306,2619-05-24 10:35:58.000774018
+-7624,2289-08-28 00:14:34
+-4279,2470-08-12 11:21:14.000955747
+-4279,2214-09-10 03:53:06
+-26998,2428-12-26 07:53:45.96925825
+17125,2629-11-15 15:34:52
+-8087,2923-07-02 11:40:26.115
+2632,2561-12-15 15:42:27
+21436,2696-05-08 05:19:24.112
+\N,2971-08-07 12:02:11.000948152
+-7624,2623-03-20 03:18:45.00006465
+-26998,2926-07-18 09:02:46.077
+11232,2507-01-27 22:04:22.49661421
+-30059,2420-12-10 22:12:30
+-15427,2355-01-08 12:34:11.617
+3198,2223-04-14 13:20:49
+-19167,2319-08-26 11:07:11.268
+14865,2220-02-28 03:41:36
+-20517,2233-12-20 04:06:56.666522799
+-15427,2046-06-07 22:58:40.728
+2748,2862-04-20 13:12:39.482805897
+-8435,2642-02-07 11:45:04.353231638
+-19167,2230-12-22 20:25:39.000242111
+-15427,2023-11-09 19:31:21
+13598,2909-06-25 23:22:50
+21436,2526-09-22 23:44:55
+-15361,2434-08-13 20:37:07.000172979
+4510,2293-01-17 13:47:41.00001006
+-8624,2120-02-15 15:36:40.000758423
+-22422,2337-07-19 06:33:02.000353352
+-26998,2268-08-04 12:48:11.848006292
+-22422,2982-12-28 06:30:26.000883228
+\N,2933-06-20 11:48:09.000839488
+3198,2736-12-20 03:59:50.343550301
+-20824,2478-11-05 00:28:05
diff --git data/files/fullouter_long_small_1c.txt data/files/fullouter_long_small_1c.txt
new file mode 100644
index 0000000000..ff323d367e
--- /dev/null
+++ data/files/fullouter_long_small_1c.txt
@@ -0,0 +1,81 @@
+-1093006502,-69.55665828
+452719211,83003.43722
+1242586043,71.1485
+-934092157,-7843850349.57130038
+294598722,-3542.6
+284554389,5.727146
+90660785,12590.288613
+-99948814,-38076694.3981
+466567142,-9763217822.129028
+1909136587,-8610.078036935181
+1242586043,-4
+\N,1.089120893565337
+1039864870,987601.57
+-466171792,0
+-1681455031,-6.4543
+1755897735,-39.965207
+1585021913,745222.66808954
+448130683,-4302.485366846491
+193709887,0.8
+-424713789,0.48
+1585021913,607.22747
+-1250662632,5454127198.951479
+294598722,-9377326244.444
+193709887,-19889.83
+1039864870,0.7
+1242586043,-749975924224.63
+-1250662632,-544.554649
+-1740848088,-9.157
+-369457052,7.7
+-369457052,560.11907883090455
+90660785,-4564.517185
+466567142,-58810.60586
+466567142,196.5785295398584
+1738753776,1525.280459649262
+1816559437,-1035.7009
+-1490239076,92253.232096
+1039864870,94.04
+560745412,678.25
+-466171792,4227.5344
+1561921421,53050.55
+-99948814,-96386.438
+1519948464,152
+1719049112,-7888197
+-793950320,-16
+-466171792,69.9
+1738753776,-99817635066320.2416
+1091836730,0.02
+891262439,-0.04
+452719211,3020.2938930744636
+-2048404259,3939387044.1
+698032489,-330457.4292625839
+-1197550983,-0.5588796922
+-2123273881,-55.89198
+-2048404259,-0.3222960446251
+1585021913,-5762331.06697112
+1785750809,47443.115
+1909136587,181.07681535944
+1801735854,-1760956929364.267
+\N,4.26165227
+1801735854,-438541294.7
+150678276,-8278
+1479580778,92077343080.7
+1091836730,-5017.14
+193709887,-0.5663
+-1681455031,-11105.372477
+-1250662632,93104
+-1197550983,0.1
+\N,682070836.2649603
+-1197550983,71852.8338674412613
+1561921421,-5.405
+-1740848088,0.506394259
+150678276,15989394.8436
+-793950320,-0.1
+-1740848088,901.441
+-477147437,6
+-1264372462,0.883
+-2123273881,3.959
+-1264372462,-6993985240226
+-1264372462,-899
+-243940373,-97176129669.654953
+-243940373,-583.258
diff --git data/files/fullouter_long_small_1d.txt data/files/fullouter_long_small_1d.txt
new file mode 100644
index 0000000000..9778d3ff62
--- /dev/null
+++ data/files/fullouter_long_small_1d.txt
@@ -0,0 +1,39 @@
+533298451
+1164387380
+1614287784
+1635405412
+-1912571616
+-894799664
+-1210744742
+-1014271154
+-747044796
+-1003639073
+436878811
+-1323620496
+-1379355738
+-1712018127
+246169862
+1431997749
+670834064
+1780951928
+-707688773
+1997943409
+1372592319
+-932176731
+162858059
+-683339273
+-497171161
+699863556
+1685473722
+41376947
+-1036083124
+1825107160
+-2038654700
+2119085509
+260588085
+-1792852276
+1831520491
+103640700
+\N
+699007128
+1840266070
diff --git data/files/fullouter_multikey_big_1a.txt data/files/fullouter_multikey_big_1a.txt
new file mode 100644
index 0000000000..fe38c7b528
--- /dev/null
+++ data/files/fullouter_multikey_big_1a.txt
@@ -0,0 +1,13 @@
+22767,-1969080993
+-17582,-1730236061
+3556,\N
+-17582,1082230084
+-17582,827141667
+1499,371855128
+-17582,9637312
+\N,1082230084
+-6131,-1969080993
+3556,-1969080993
+\N,\N
+-18222,-1969080993
+-17582,267529350
diff --git data/files/fullouter_multikey_big_1a_nonull.txt data/files/fullouter_multikey_big_1a_nonull.txt
new file mode 100644
index 0000000000..40e84b04be
--- /dev/null
+++ data/files/fullouter_multikey_big_1a_nonull.txt
@@ -0,0 +1,10 @@
+-17582,1082230084
+22767,-1969080993
+-17582,827141667
+-17582,-1730236061
+3556,-1969080993
+-6131,-1969080993
+-18222,-1969080993
+1499,371855128
+-17582,267529350
+-17582,9637312
diff --git data/files/fullouter_multikey_big_1b.txt data/files/fullouter_multikey_big_1b.txt
new file mode 100644
index 0000000000..40cfb9a954
--- /dev/null
+++ data/files/fullouter_multikey_big_1b.txt
@@ -0,0 +1,17 @@
+2061-12-19 22:10:32.000628309,21635,ANCO
+\N,21635,ANCO
+2686-05-23 07:46:46.565832918,13212,NCYBDW
+2082-07-14 04:00:40.695380469,12556,NCYBDW
+2188-06-04 15:03:14.963259704,9468,AAA
+2608-02-23 23:44:02.546440891,26184,NCYBDW
+2093-04-10 23:36:54.846,\N,\N
+2898-10-01 22:27:02.000871113,10361,NCYBDW
+2306-06-21 11:02:00.143124239,1446,\N
+\N,-6909,\N
+\N,\N,\N
+2306-06-21 11:02:00.143124239,-6909,NCYBDW
+2093-04-10 23:36:54.846,1446,GHZVPWFO
+\N,\N,CCWYD
+2686-05-23 07:46:46.565832918,\N,GHZVPWFO
+2093-04-10 23:36:54.846,28996,Q
+2299-11-15 16:41:30.401,-31077,NCYBDW
diff --git data/files/fullouter_multikey_small_1a.txt data/files/fullouter_multikey_small_1a.txt
new file mode 100644
index 0000000000..4e0742c8b8
--- /dev/null
+++ data/files/fullouter_multikey_small_1a.txt
@@ -0,0 +1,92 @@
+23015,258882280
+23015,-276888585
+21186,-586336015
+-22311,-2055239583
+3412,-1249487623
+\N,1082230084
+20156,-1618478138
+-17788,-738743861
+-24206,-1456409156
+30353,2044473567
+20969,-1995259010
+-23457,-63842445
+3412,-2081156563
+-6131,-1969080993
+23015,-252525791
+30353,1364268303
+23015,564751472
+15404,1078466156
+4586,-586336015
+-4117,-1386947816
+-26894,-63842445
+-17788,-1361776766
+-7386,-2112062470
+23015,-1893013623
+30353,1241923267
+-24206,641361618
+-28129,-2055239583
+-20125,-1995259010
+16166,931172175
+31443,-1968665833
+-28313,837320573
+11460,1078466156
+15061,-63842445
+13672,-63842445
+14400,-825652334
+-7386,100736776
+26944,-1995259010
+-11868,97203778
+12089,-63842445
+-28137,-63842445
+3412,1253976194
+-980,2009785365
+16696,-63842445
+-11868,930596435
+4902,1078466156
+-17582,267529350
+-12252,964377504
+20156,963883665
+-11868,1658440922
+4779,-1995259010
+-7386,-1635102480
+-28313,51228026
+-11868,1052120431
+-980,-270600267
+-20900,1078466156
+\N,\N
+20156,1165375499
+30353,-1507157031
+3412,-1196037018
+22934,-1695419330
+30353,105613996
+-17788,-872691214
+-980,-333603940
+30353,-1011627089
+-11868,-3536499
+-2407,1078466156
+23015,-217613200
+-28313,-706104224
+-980,712692345
+-11868,1456809245
+-17788,528419995
+-11868,-915441041
+-980,628784462
+30353,-1007182618
+23015,-696928205
+-980,356970043
+23015,-893234501
+-980,-465544127
+-5734,1078466156
+-980,-801821285
+26738,-2055239583
+8177,-1995259010
+-11868,1318114822
+3890,1411429004
+-6061,-586336015
+3412,-2132472060
+-15212,-2055239583
+-12252,1956403781
+5957,-1995259010
+-1787,-63842445
+20156,1855042153
+-980,1310479628
diff --git data/files/fullouter_multikey_small_1a_nonull.txt data/files/fullouter_multikey_small_1a_nonull.txt
new file mode 100644
index 0000000000..2a8b9a1d90
--- /dev/null
+++ data/files/fullouter_multikey_small_1a_nonull.txt
@@ -0,0 +1,90 @@
+16696,-63842445
+4586,-586336015
+26738,-2055239583
+-17788,-738743861
+-28313,-706104224
+-23457,-63842445
+-20900,1078466156
+-12252,964377504
+-28313,51228026
+-11868,-3536499
+11460,1078466156
+26944,-1995259010
+20156,1855042153
+-11868,97203778
+15061,-63842445
+-17788,528419995
+-26894,-63842445
+-28313,837320573
+20156,963883665
+-15212,-2055239583
+5957,-1995259010
+30353,-1011627089
+3890,1411429004
+-980,-333603940
+13672,-63842445
+-980,628784462
+23015,-252525791
+-11868,1052120431
+-980,356970043
+23015,-217613200
+-6061,-586336015
+-5734,1078466156
+-11868,1318114822
+23015,258882280
+-2407,1078466156
+12089,-63842445
+3412,-2132472060
+-28129,-2055239583
+-980,-270600267
+16166,931172175
+-7386,100736776
+4902,1078466156
+20969,-1995259010
+22934,-1695419330
+3412,-1249487623
+3412,1253976194
+21186,-586336015
+8177,-1995259010
+-7386,-1635102480
+-11868,1456809245
+-20125,-1995259010
+-980,-801821285
+-980,1310479628
+23015,564751472
+23015,-893234501
+4779,-1995259010
+-980,2009785365
+-24206,641361618
+30353,-1507157031
+14400,-825652334
+3412,-2081156563
+20156,-1618478138
+31443,-1968665833
+-22311,-2055239583
+30353,1241923267
+-11868,930596435
+-17788,-1361776766
+-24206,-1456409156
+-7386,-2112062470
+30353,1364268303
+23015,-1893013623
+-17788,-872691214
+30353,2044473567
+-28137,-63842445
+30353,105613996
+-6131,-1969080993
+-17582,267529350
+23015,-276888585
+-12252,1956403781
+23015,-696928205
+-11868,1658440922
+-1787,-63842445
+-11868,-915441041
+-980,-465544127
+30353,-1007182618
+-980,712692345
+20156,1165375499
+3412,-1196037018
+15404,1078466156
+-4117,-1386947816
diff --git data/files/fullouter_multikey_small_1b.txt data/files/fullouter_multikey_small_1b.txt
new file mode 100644
index 0000000000..b56a3f7f46
--- /dev/null
+++ data/files/fullouter_multikey_small_1b.txt
@@ -0,0 +1,118 @@
+2304-12-15 15:31:16,11101,YJCKKCR,-0.2
+2018-11-25 22:27:55.84,-12202,VBDBM,7506645.9537
+1957-03-06 09:57:31,-26373,NXLNNSO,2
+2332-06-14 07:02:42.32,-26373,XFFFDTQ,56845106806308.9
+2535-03-01 05:04:49.000525883,23663,ALIQKNXHE,-0.1665691
+2629-04-07 01:54:11,-6776,WGGFVFTW,6.8012851708
+2266-09-26 06:27:29.000284762,20223,EDYJJN,14
+2969-01-23 14:08:04.000667259,-18138,VDPN,8924831210.42768019
+2861-05-27 07:13:01.000848622,-19598,WKPXNLXS,29399
+2301-06-03 17:16:19,15332,ZVEUKC,0.5
+1980-09-13 19:57:15,\N,M,57650.7723
+2304-12-15 15:31:16,1301,T,-0.8
+2461-03-09 09:54:45.000982385,-16454,ZSMB,-991.43605
+2044-05-02 07:00:03.35,-8751,ZSMB,-453797242.029791752
+2409-09-23 10:33:27,2638,XSXR,-9926693851
+1941-10-16 02:19:36.000423663,-24459,AO,-821445414.4579712
+2512-10-06 03:03:03,-3465,VZQ,-49.51219
+2971-02-14 09:13:19,-16605,BVACIRP,-5.751278023
+2075-10-25 20:32:40.000792874,\N,\N,226612651968.36076
+2073-03-21 15:32:57.617920888,26425,MPRACIRYW,5
+2969-01-23 14:08:04.000667259,14500,WXLTRFQP,-23.8198
+2898-12-18 03:37:17,-24459,MHNBXPBM,14.23669356238481
+\N,\N,\N,-2207.3
+2391-01-17 15:28:37.00045143,16160,ZVEUKC,771355639420297.133
+2309-01-15 12:43:49,22821,ZMY,40.9
+2340-12-15 05:15:17.133588982,23663,HHTP,33383.8
+2969-01-23 14:08:04.000667259,-8913,UIMQ,9.178
+2145-10-15 06:58:42.831,2638,\N,-9784.82
+2888-05-08 08:36:55.182302102,5786,ZVEUKC,-56082455.033918
+2467-05-11 06:04:13.426693647,23196,EIBSDASR,-8.5548883801
+2829-06-04 08:01:47.836,22771,ZVEUKC,94317.75318
+2938-12-21 23:35:59.498,29362,ZMY,0.88
+2304-12-15 15:31:16,-13125,JFYW,6.086657
+2808-07-09 02:10:11.928498854,-19598,FHFX,0.3
+2083-06-07 09:35:19.383,-26373,MR,-394.0867
+2686-05-23 07:46:46.565832918,13212,NCYBDW,-917116793.4
+2969-01-23 14:08:04.000667259,-8913,UIMQ,-375994644577.315257
+2338-02-12 09:30:07,20223,CTH,-6154.763054
+2629-04-07 01:54:11,-6776,WGGFVFTW,41.77451507786646
+2242-08-04 07:51:46.905,20223,UCYXACQ,37.7288
+2637-03-12 22:25:46.385,-12923,PPTJPFR,5.4
+2304-12-15 15:31:16,8650,RLNO,0.71351747335
+2688-02-06 20:58:42.000947837,20223,PAIY,67661.735
+\N,\N,\N,-2.4
+2512-10-06 03:03:03,-3465,VZQ,0.4458
+2960-04-12 07:03:42.000366651,20340,CYZYUNSF,-96.3
+2461-03-09 09:54:45.000982385,-16454,ZSMB,-9575827.55396
+2512-10-06 03:03:03,1560,X,-922.6951584107
+2396-04-06 15:39:02.404013577,29661,ZSMB,0.76718326
+2409-09-23 10:33:27,2638,XSXR,0.4
+2969-01-23 14:08:04.000667259,6689,TFGVOGPJF,-0.01
+2333-07-28 09:59:26,23196,RKSK,37872288434740893.5
+2409-09-23 10:33:27,2638,XSXR,-162.95
+2357-05-08 07:09:09.000482799,6226,ZSMB,-472
+2304-12-15 15:31:16,15090,G,-4319470286240016.3
+2304-12-15 15:31:16,1301,T,61.302
+2105-01-04 16:27:45,23100,ZSMB,-83.2328
+2242-08-04 07:51:46.905,20223,UCYXACQ,-0.26149
+2637-03-12 22:25:46.385,-17786,HYEGQ,-84.169614329419
+1931-12-04 11:13:47.269597392,23196,HVJCQMTQL,-9697532.8994
+2897-08-10 15:21:47.09,23663,XYUVBED,6370
+2888-05-08 08:36:55.182302102,5786,ZVEUKC,57.62175257788037
+2145-10-15 06:58:42.831,2638,UANGISEXR,-5996.306
+2462-12-16 23:11:32.633305644,-26373,CB,67.41799
+2396-04-06 15:39:02.404013577,29661,ZSMB,-5151598.347
+2304-12-15 15:31:16,15090,G,975
+2512-10-06 03:03:03,32099,ARNZ,-0.41
+2188-06-04 15:03:14.963259704,9468,AAA,2.75496352
+2512-10-06 03:03:03,1560,X,761196.522
+2304-12-15 15:31:16,1301,T,2720.8
+1919-06-20 00:16:50.611028595,20223,ZKBC,-23
+2897-08-10 15:21:47.09,23663,XYUVBED,51.7323303273
+2086-04-09 00:03:10,20223,THXNJGFFV,-85184687349898.892
+2238-05-17 19:27:25.519,20223,KQCM,-0.01095
+2086-04-09 00:03:10,20223,THXNJGFFV,482.5383411359219
+2480-10-02 09:31:37.000770961,-26373,NBN,-5875.5197252
+2086-04-09 00:03:10,20223,THXNJGFFV,0.4396861
+2759-11-26 22:19:55.410967136,-27454,ZMY,60.6025797
+2083-06-07 09:35:19.383,-26373,MR,67892053.02376094
+2882-05-20 07:21:25.221299462,23196,U,-9951044
+2971-02-14 09:13:19,-16605,BVACIRP,-27394351.3
+2512-10-06 03:03:03,24313,QBHUG,-8423.151573236
+2882-05-20 07:21:25.221299462,23196,U,-4244.926206619
+1905-04-20 13:42:25.000469776,2638,KAUUFF,7
+2410-05-03 13:44:56,2638,PHOR,-769088.176482
+2668-06-25 07:12:37.000970744,2638,TJE,-2.7796827
+2969-01-23 14:08:04.000667259,-32485,AGEPWWLJF,-48431309405.652522
+2410-05-03 13:44:56,2638,PHOR,93262.914526611
+2512-10-06 03:03:03,13195,CRJ,14
+2018-11-25 22:27:55.84,-12202,VBDBM,98790.713907420831
+2304-12-15 15:31:16,8650,RLNO,-0.4355
+2071-07-21 20:02:32.000250697,2638,NRUV,-66198.351092
+2525-05-12 15:59:35,-24459,SAVRGA,53106747151.8633
+2637-03-12 22:25:46.385,21841,CXTI,749563668434009.65
+2018-11-25 22:27:55.84,-22419,LOTLS,342.3726040228584
+2637-03-12 22:25:46.385,21841,CXTI,7362887891522.3782
+2038-10-12 09:15:33.000539653,-19598,YKNIAJW,-642807895924.66
+2957-05-07 10:41:46,20223,OWQT,-586953.153681
+2304-12-15 15:31:16,11101,YJCKKCR,1279917802.42
+2355-09-23 19:52:34.638084141,-19598,H,92.15
+2960-04-12 07:03:42.000366651,20340,CYZYUNSF,2.1577659
+2355-09-23 19:52:34.638084141,-19598,H,74179461.880493
+2969-01-23 14:08:04.000667259,-8913,UIMQ,-81
+\N,-12914,ZVEUKC,221
+2743-12-27 05:16:19.000573579,-12914,ZVEUKC,-811984611.5178497
+1957-02-01 14:00:29.000548421,-16085,ZVEUKC,-2312.8149
+2201-07-05 17:22:06.084206844,-24459,UBGT,1.5069483282
+2461-03-09 09:54:45.000982385,-16454,ZSMB,8694.89
+2169-04-02 06:30:32,23855,PDVQATOS,-1515597428
+2304-12-15 15:31:16,30285,GSJPSIYOU,0.2
+2913-07-17 15:06:58.041,-10206,\N,-0.2
+2169-04-02 06:30:32,23855,PDVQATOS,-4016.9608
+2759-11-26 22:19:55.410967136,-27454,ZMY,368
+2073-03-21 15:32:57.617920888,26425,MPRACIRYW,726945733.4193
+2304-12-15 15:31:16,11101,YJCKKCR,-0.5
+2462-12-16 23:11:32.633305644,-26373,CB,-582687
+2357-05-08 07:09:09.000482799,6226,ZSMB,-32.46
+2304-12-15 15:31:16,12587,OPW,-4.59489504
diff --git data/files/fullouter_string_big_1a.txt data/files/fullouter_string_big_1a.txt
new file mode 100644
index 0000000000..1cbcd05d26
--- /dev/null
+++ data/files/fullouter_string_big_1a.txt
@@ -0,0 +1,13 @@
+FTWURVH
+QNCYBDW
+UA
+WXHJ
+\N
+WXHJ
+PXLD
+WXHJ
+PXLD
+WXHJ
+WXHJ
+MXGDMBD
+PXLD
diff --git data/files/fullouter_string_big_1a_nonull.txt data/files/fullouter_string_big_1a_nonull.txt
new file mode 100644
index 0000000000..a6566f2916
--- /dev/null
+++ data/files/fullouter_string_big_1a_nonull.txt
@@ -0,0 +1,12 @@
+WXHJ
+WXHJ
+FTWURVH
+MXGDMBD
+UA
+WXHJ
+QNCYBDW
+PXLD
+PXLD
+WXHJ
+PXLD
+WXHJ
diff --git data/files/fullouter_string_big_1a_old.txt data/files/fullouter_string_big_1a_old.txt
new file mode 100644
index 0000000000..1fa51ad799
--- /dev/null
+++ data/files/fullouter_string_big_1a_old.txt
@@ -0,0 +1,13 @@
+WXHJ
+WXHJ
+WXHJ
+WXHJ
+WXHJ
+QNCYBDW
+PXLD
+PXLD
+PXLD
+UA
+\N
+FTWURVH
+MXGDMBD
diff --git data/files/fullouter_string_small_1a.txt data/files/fullouter_string_small_1a.txt
new file mode 100644
index 0000000000..f223da07e2
--- /dev/null
+++ data/files/fullouter_string_small_1a.txt
@@ -0,0 +1,38 @@
+BDBMW,2278-04-27,2101-02-21 08:53:34.692
+FROPIK,2023-02-28,2467-05-11 06:04:13.426693647
+GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302
+MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592
+CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094
+,1985-01-22,2111-01-10 15:44:28
+IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888
+GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251
+\N,1865-11-08,2893-04-07 07:36:12
+BEP,2206-08-10,2331-10-09 10:59:51
+NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922
+\N,2250-04-22,2548-03-21 08:23:13.133573801
+ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799
+IWEZJHKE,\N,\N
+AARNZRVZQ,2002-10-23,2525-05-12 15:59:35
+BEP,2141-02-19,2521-06-09 01:20:07.121
+AARNZRVZQ,2000-11-13,2309-06-05 19:54:13
+LOTLS,1957-11-09,2092-06-07 06:42:30.000538454
+FROPIK,2124-10-01,2974-07-06 12:05:08.000146048
+KL,1980-09-22,2073-08-25 11:51:10.318
+\N,1915-02-22,2554-10-27 09:34:30
+WNGFTTY,1843-06-10,2411-01-28 20:03:59
+VNRXWQ,1883-02-06,2287-07-17 16:46:58.287
+QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266
+GOYJHW,1959-04-27,\N
+LOTLS,2099-08-04,2181-01-25 01:04:25.000030055
+CQMTQLI,2090-11-13,2693-03-17 16:19:55.82
+VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887
+LOTLS,2126-09-16,1977-12-15 15:28:56
+FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556
+,2021-02-21,2802-04-21 18:48:18.5933838
+ZNOUDCR,\N,1988-04-23 08:40:21
+FROPIK,2214-02-09,1949-08-18 17:14:38.000703738
+SDA,2196-04-12,2462-10-26 19:28:12.733
+WNGFTTY,2251-08-16,2649-12-21 18:30:42.498
+GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784
+FYW,1807-03-20,2305-08-17 01:32:44
+ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558
diff --git data/files/fullouter_string_small_1a_nonull.txt data/files/fullouter_string_small_1a_nonull.txt
new file mode 100644
index 0000000000..6b97ef4a1d
--- /dev/null
+++ data/files/fullouter_string_small_1a_nonull.txt
@@ -0,0 +1,35 @@
+LOTLS,2126-09-16,1977-12-15 15:28:56
+MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592
+WNGFTTY,2251-08-16,2649-12-21 18:30:42.498
+QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266
+AARNZRVZQ,2002-10-23,2525-05-12 15:59:35
+BEP,2141-02-19,2521-06-09 01:20:07.121
+ZNOUDCR,\N,1988-04-23 08:40:21
+FROPIK,2023-02-28,2467-05-11 06:04:13.426693647
+GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784
+CQMTQLI,2090-11-13,2693-03-17 16:19:55.82
+BDBMW,2278-04-27,2101-02-21 08:53:34.692
+AARNZRVZQ,2000-11-13,2309-06-05 19:54:13
+FYW,1807-03-20,2305-08-17 01:32:44
+,2021-02-21,2802-04-21 18:48:18.5933838
+VNRXWQ,1883-02-06,2287-07-17 16:46:58.287
+FROPIK,2124-10-01,2974-07-06 12:05:08.000146048
+LOTLS,2099-08-04,2181-01-25 01:04:25.000030055
+BEP,2206-08-10,2331-10-09 10:59:51
+WNGFTTY,1843-06-10,2411-01-28 20:03:59
+LOTLS,1957-11-09,2092-06-07 06:42:30.000538454
+CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094
+GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302
+,1985-01-22,2111-01-10 15:44:28
+SDA,2196-04-12,2462-10-26 19:28:12.733
+ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799
+GOYJHW,1959-04-27,\N
+FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556
+KL,1980-09-22,2073-08-25 11:51:10.318
+ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558
+NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922
+FROPIK,2214-02-09,1949-08-18 17:14:38.000703738
+IWEZJHKE,\N,\N
+GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251
+IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888
+VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887
diff --git data/files/fullouter_string_small_1a_old.txt data/files/fullouter_string_small_1a_old.txt
new file mode 100644
index 0000000000..505c4032e5
--- /dev/null
+++ data/files/fullouter_string_small_1a_old.txt
@@ -0,0 +1,38 @@
+,2021-02-21,2802-04-21 18:48:18.5933838
+,1985-01-22,2111-01-10 15:44:28
+VNRXWQ,1883-02-06,2287-07-17 16:46:58.287
+VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887
+KL,1980-09-22,2073-08-25 11:51:10.318
+FYW,1807-03-20,2305-08-17 01:32:44
+WNGFTTY,2251-08-16,2649-12-21 18:30:42.498
+WNGFTTY,1843-06-10,2411-01-28 20:03:59
+FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556
+CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094
+CQMTQLI,2090-11-13,2693-03-17 16:19:55.82
+BEP,2141-02-19,2521-06-09 01:20:07.121
+BEP,2206-08-10,2331-10-09 10:59:51
+FROPIK,2023-02-28,2467-05-11 06:04:13.426693647
+FROPIK,2214-02-09,1949-08-18 17:14:38.000703738
+FROPIK,2124-10-01,2974-07-06 12:05:08.000146048
+SDA,2196-04-12,2462-10-26 19:28:12.733
+ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558
+ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799
+MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592
+IWEZJHKE,\N,\N
+NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922
+GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784
+GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302
+GOYJHW,1959-04-27,\N
+QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266
+IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888
+ZNOUDCR,\N,1988-04-23 08:40:21
+AARNZRVZQ,2000-11-13,2309-06-05 19:54:13
+AARNZRVZQ,2002-10-23,2525-05-12 15:59:35
+BDBMW,2278-04-27,2101-02-21 08:53:34.692
+\N,1865-11-08,2893-04-07 07:36:12
+\N,2250-04-22,2548-03-21 08:23:13.133573801
+\N,1915-02-22,2554-10-27 09:34:30
+LOTLS,1957-11-09,2092-06-07 06:42:30.000538454
+LOTLS,2126-09-16,1977-12-15 15:28:56
+LOTLS,2099-08-04,2181-01-25 01:04:25.000030055
+GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251
diff --git hcatalog/hcatalog-pig-adapter/pom.xml hcatalog/hcatalog-pig-adapter/pom.xml
index a1c8ddf057..c026835f52 100644
--- hcatalog/hcatalog-pig-adapter/pom.xml
+++ hcatalog/hcatalog-pig-adapter/pom.xml
@@ -136,6 +136,13 @@
tests
test
+
+ org.apache.hive
+ hive-standalone-metastore-server
+ 4.0.0-SNAPSHOT
+ tests
+ test
+
org.apache.hadoop
hadoop-mapreduce-client-common
diff --git hcatalog/pom.xml hcatalog/pom.xml
index 4894e9a16f..91d7dafa9c 100644
--- hcatalog/pom.xml
+++ hcatalog/pom.xml
@@ -65,6 +65,13 @@
${hadoop.version}
test
+
+ org.apache.hive
+ hive-standalone-metastore-server
+ 4.0.0-SNAPSHOT
+ tests
+ test
+
org.apache.pig
pig
diff --git itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
index b6fff6048f..eb08d5753c 100644
--- itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
+++ itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
@@ -150,7 +150,7 @@ STAGE PLANS:
GatherStats: false
Reduce Output Operator
key expressions: _col0 (type: int)
- null sort order: a
+ null sort order: z
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
tag: -1
@@ -425,7 +425,7 @@ STAGE PLANS:
GatherStats: false
Reduce Output Operator
key expressions: _col0 (type: int)
- null sort order: a
+ null sort order: z
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
tag: -1
diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java
index af446dbcbc..48f85f4610 100644
--- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java
+++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java
@@ -13,21 +13,25 @@
*/
package org.apache.hive.benchmark.vectorization.mapjoin;
+import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.tez.ObjectCache;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
import org.apache.hadoop.hive.ql.exec.util.collectoroperator.CountCollectorTestOperator;
import org.apache.hadoop.hive.ql.exec.util.collectoroperator.CountVectorCollectorTestOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.MapJoinTestImplementation;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestData;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.CreateMapJoinResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription.MapJoinPlanVariation;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription.SmallTableGenerationParameters;
-import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerateUtil;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.VectorMapJoinVariation;
@@ -74,30 +78,31 @@ public void bench() throws Exception {
}
protected void setupMapJoin(HiveConf hiveConf, long seed, int rowCount,
- VectorMapJoinVariation vectorMapJoinVariation, MapJoinTestImplementation mapJoinImplementation,
- String[] bigTableColumnNames, TypeInfo[] bigTableTypeInfos, int[] bigTableKeyColumnNums,
- String[] smallTableValueColumnNames, TypeInfo[] smallTableValueTypeInfos,
- int[] bigTableRetainColumnNums,
- int[] smallTableRetainKeyColumnNums, int[] smallTableRetainValueColumnNums,
- SmallTableGenerationParameters smallTableGenerationParameters) throws Exception {
+ VectorMapJoinVariation vectorMapJoinVariation, MapJoinTestImplementation mapJoinImplementation,
+ String[] bigTableColumnNames, TypeInfo[] bigTableTypeInfos,
+ int[] bigTableKeyColumnNums,
+ String[] smallTableValueColumnNames, TypeInfo[] smallTableValueTypeInfos,
+ int[] bigTableRetainColumnNums,
+ int[] smallTableRetainKeyColumnNums, int[] smallTableRetainValueColumnNums,
+ SmallTableGenerationParameters smallTableGenerationParameters) throws Exception {
this.vectorMapJoinVariation = vectorMapJoinVariation;
this.mapJoinImplementation = mapJoinImplementation;
testDesc = new MapJoinTestDescription(
hiveConf, vectorMapJoinVariation,
- bigTableColumnNames, bigTableTypeInfos,
+ bigTableTypeInfos,
bigTableKeyColumnNums,
- smallTableValueColumnNames, smallTableValueTypeInfos,
- bigTableRetainColumnNums,
- smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums,
- smallTableGenerationParameters);
+ smallTableValueTypeInfos,
+ smallTableRetainKeyColumnNums,
+ smallTableGenerationParameters,
+ MapJoinPlanVariation.DYNAMIC_PARTITION_HASH_JOIN);
// Prepare data. Good for ANY implementation variation.
- testData = new MapJoinTestData(rowCount, testDesc, seed, seed * 10);
+ testData = new MapJoinTestData(rowCount, testDesc, seed);
ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl();
ObjectCache.setupObjectRegistry(objectRegistry);
-
+
operator = setupBenchmarkImplementation(
mapJoinImplementation, testDesc, testData);
@@ -108,15 +113,21 @@ protected void setupMapJoin(HiveConf hiveConf, long seed, int rowCount,
*/
if (!isVectorOutput) {
- bigTableRows = VectorBatchGenerateUtil.generateRowObjectArray(
- testDesc.bigTableKeyTypeInfos, testData.getBigTableBatchStream(),
- testData.getBigTableBatch(), testDesc.outputObjectInspectors);
+ bigTableRows = testData.getBigTableBatchSource().getRandomRows();
} else {
- bigTableBatches = VectorBatchGenerateUtil.generateBatchArray(
- testData.getBigTableBatchStream(), testData.getBigTableBatch());
-
+ ArrayList bigTableBatchList = new ArrayList();
+ VectorRandomBatchSource batchSource = testData.getBigTableBatchSource();
+ batchSource.resetBatchIteration();
+ while (true) {
+ VectorizedRowBatch batch = testData.createBigTableBatch(testDesc);
+ if (!batchSource.fillNextBatch(batch)) {
+ break;
+ }
+ bigTableBatchList.add(batch);
+ }
+ bigTableBatches = bigTableBatchList.toArray(new VectorizedRowBatch[0]);
}
}
@@ -131,7 +142,6 @@ protected static MapJoinOperator setupBenchmarkImplementation(
MapJoinTestData testData)
throws Exception {
- // UNDONE: Parameterize for implementation variation?
MapJoinDesc mapJoinDesc = MapJoinTestConfig.createMapJoinDesc(testDesc);
final boolean isVectorOutput = isVectorOutput(mapJoinImplementation);
@@ -141,9 +151,19 @@ protected static MapJoinOperator setupBenchmarkImplementation(
(!isVectorOutput ? new CountCollectorTestOperator() :
new CountVectorCollectorTestOperator());
- MapJoinOperator operator =
+ CreateMapJoinResult createMapJoinResult =
MapJoinTestConfig.createMapJoinImplementation(
- mapJoinImplementation, testDesc, testCollectorOperator, testData, mapJoinDesc);
+ mapJoinImplementation, testDesc, testData, mapJoinDesc,
+ /* shareMapJoinTableContainer */ null);
+ MapJoinOperator operator = createMapJoinResult.mapJoinOperator;
+ MapJoinTableContainer mapJoinTableContainer = createMapJoinResult.mapJoinTableContainer;
+
+ // Invoke initializeOp methods.
+ operator.initialize(testDesc.hiveConf, testDesc.inputObjectInspectors);
+
+ // Fixup the mapJoinTables.
+ operator.setTestMapJoinTableContainer(1, mapJoinTableContainer, null);
+
return operator;
}
diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java
index c9da92a754..aa882973ec 100644
--- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java
+++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java
@@ -59,7 +59,8 @@ public void doSetup(VectorMapJoinVariation vectorMapJoinVariation,
setupMapJoin(hiveConf, seed, rowCount,
vectorMapJoinVariation, mapJoinImplementation,
- bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums,
+ bigTableColumnNames, bigTableTypeInfos,
+ bigTableKeyColumnNums,
smallTableValueColumnNames, smallTableValueTypeInfos,
bigTableRetainColumnNums,
smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums,
diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java
index a6b47192a8..60b28907a5 100644
--- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java
+++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java
@@ -57,7 +57,8 @@ public void doSetup(VectorMapJoinVariation vectorMapJoinVariation,
setupMapJoin(hiveConf, seed, rowCount,
vectorMapJoinVariation, mapJoinImplementation,
- bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums,
+ bigTableColumnNames, bigTableTypeInfos,
+ bigTableKeyColumnNums,
smallTableValueColumnNames, smallTableValueTypeInfos,
bigTableRetainColumnNums,
smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums,
diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java
index 1b310385d2..937ede1882 100644
--- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java
+++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java
@@ -57,7 +57,8 @@ public void doSetup(VectorMapJoinVariation vectorMapJoinVariation,
setupMapJoin(hiveConf, seed, rowCount,
vectorMapJoinVariation, mapJoinImplementation,
- bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums,
+ bigTableColumnNames, bigTableTypeInfos,
+ bigTableKeyColumnNums,
smallTableValueColumnNames, smallTableValueTypeInfos,
bigTableRetainColumnNums,
smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums,
diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 6c45641a0d..35ad982f20 100644
--- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1400,4 +1400,25 @@ public void testDumpExternalTableSetTrue() throws Throwable {
.run("select id from t4")
.verifyResult(null); // Returns null as create table event doesn't list files
}
+
+ @Test
+ public void testDumpExternalTableWithAddPartitionEvent() throws Throwable {
+ WarehouseInstance.Tuple tuple = primary.dump("repl dump " + primaryDbName);
+
+ replica.load(replicatedDbName, tuple.dumpLocation);
+
+ tuple = primary.run("use " + primaryDbName)
+ .run("create external table t1 (place string) partitioned by (country string)")
+ .run("alter table t1 add partition(country='india')")
+ .run("alter table t1 add partition(country='us')")
+ .dump("repl dump " + primaryDbName + " from " + tuple.lastReplicationId
+ + " with ('hive.repl.include.external.tables'='true')");
+
+ replica.load(replicatedDbName, tuple.dumpLocation)
+ .run("use " + replicatedDbName)
+ .run("show tables like 't1'")
+ .verifyResult("t1")
+ .run("show partitions t1")
+ .verifyResults(new String[] { "country=india", "country=us" });
+ }
}
diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java
deleted file mode 100644
index 4768975225..0000000000
--- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.cli;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import java.io.File;
-import java.util.List;
-
-@RunWith(Parameterized.class)
-public class TestMiniDruidKafkaCliDriver {
-
- static CliAdapter adapter = new CliConfigs.MiniDruidKafkaCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List
-
- org.apache.hive
- hive-standalone-metastore-server
- ${project.version}
javolution
diff --git ql/pom.xml ql/pom.xml
index a55cbe380d..d73deba440 100644
--- ql/pom.xml
+++ ql/pom.xml
@@ -448,6 +448,11 @@
opencsv
${opencsv.version}
+
+ org.apache.hive
+ hive-standalone-metastore-server
+ ${project.version}
+
org.apache.hive
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
index 02a67cbd39..3762ee5ab0 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
@@ -790,7 +790,16 @@ private boolean hasRightPairForLeft(int left, int right) {
}
private boolean hasAnyFiltered(int alias, List