diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index 27b353c..a589f33 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -151,6 +151,10 @@ // Indicates if this instance of beeline is running in compatibility mode, or beeline mode private boolean isBeeLine = true; + // Indicates that we are in test mode. + // Print only the errors, the operation log and the query results. + private boolean isTestMode = false; + private static final Options options = new Options(); public static final String BEELINE_DEFAULT_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver"; @@ -2438,4 +2442,19 @@ public String getCurrentDatabase() { public void setCurrentDatabase(String currentDatabase) { this.currentDatabase = currentDatabase; } + + /** + * Setting the BeeLine into test mode. + * Print only the errors, the operation log and the query results. + * Should be used only by tests. + * + * @param isTestMode + */ + void setIsTestMode(boolean isTestMode) { + this.isTestMode = isTestMode; + } + + boolean isTestMode() { + return isTestMode; + } } diff --git beeline/src/java/org/apache/hive/beeline/Commands.java beeline/src/java/org/apache/hive/beeline/Commands.java index d179b37..08d53ca 100644 --- beeline/src/java/org/apache/hive/beeline/Commands.java +++ beeline/src/java/org/apache/hive/beeline/Commands.java @@ -977,7 +977,8 @@ private boolean executeInternal(String sql, boolean call) { hasResults = ((CallableStatement) stmnt).execute(); } else { stmnt = beeLine.createStatement(); - if (beeLine.getOpts().isSilent()) { + // In test mode we want the operation logs regardless of the settings + if (!beeLine.isTestMode() && beeLine.getOpts().isSilent()) { hasResults = stmnt.execute(sql); } else { InPlaceUpdateStream.EventNotifier eventNotifier = @@ -1341,7 +1342,12 @@ private void updateQueryLog() { try { List queryLogs = hiveStatement.getQueryLog(); for (String log : queryLogs) { - commands.beeLine.info(log); + if (!commands.beeLine.isTestMode()) { + commands.beeLine.info(log); + } else { + // In test mode print the logs to the output + commands.beeLine.output(log); + } } if (!queryLogs.isEmpty()) { notifier.operationLogShowedToUser(); @@ -1385,7 +1391,12 @@ private void showRemainingLogsIfAny(Statement statement) { return; } for (String log : logs) { - beeLine.info(log); + if (!beeLine.isTestMode()) { + beeLine.info(log); + } else { + // In test mode print the logs to the output + beeLine.output(log); + } } } while (logs.size() > 0); } else { diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 726522b..420d35e 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -548,6 +548,13 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "If not set, defaults to the codec extension for text files (e.g. \".gz\"), or no extension otherwise."), HIVE_IN_TEST("hive.in.test", false, "internal usage only, true in test mode", true), + HIVE_IN_TEST_SHORT_LOGS("hive.in.test.short.logs", false, + "internal usage only, used only in test mode. If set true, when requesting the " + + "operation logs the short version (generated by LogDivertAppenderForTest) will be " + + "returned"), + HIVE_IN_TEST_REMOVE_LOGS("hive.in.test.remove.logs", true, + "internal usage only, used only in test mode. If set false, the operation logs, and the " + + "operation log directory will not be removed, so they can be found after the test runs."), HIVE_IN_TEZ_TEST("hive.in.tez.test", false, "internal use only, true when in testing tez", true), diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index df80478..2551d0a 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -756,8 +756,8 @@ beeline.positive.include=drop_with_concurrency.q,\ smb_mapjoin_16.q,\ smb_mapjoin_2.q,\ smb_mapjoin_3.q,\ - smb_mapjoin_7.q - + smb_mapjoin_7.q,\ + select_dummy_source.q minimr.query.negative.files=cluster_tasklog_retrieval.q,\ file_with_header_footer_negative.q,\ diff --git itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java itests/util/src/main/java/org/apache/hive/beeline/QFile.java similarity index 80% rename from itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java rename to itests/util/src/main/java/org/apache/hive/beeline/QFile.java index 9fae194..0bde529 100644 --- itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java +++ itests/util/src/main/java/org/apache/hive/beeline/QFile.java @@ -59,6 +59,8 @@ private static final Pattern USE_PATTERN = Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE); + private static final String MASK_PATTERN = "#### A masked pattern was here ####\n"; + private String name; private File inputFile; private File rawOutputFile; @@ -67,7 +69,8 @@ private File logFile; private File beforeExecuteLogFile; private File afterExecuteLogFile; - private static RegexFilterSet filterSet = getFilterSet(); + private static RegexFilterSet staticFilterSet = getStaticFilterSet(); + private RegexFilterSet specificFilterSet; private boolean rewriteSourceTables; private QFile() {} @@ -146,9 +149,26 @@ private String replaceTableNames(String source) { return source; } + /** + * The result contains the original queries. To revert them to the original form remove the + * 'default' from every default.TABLE_NAME, like default.src->src, default.srcpart->srcpart. + * @param source The original query output + * @return The query output where the tablenames are replaced + */ + private String revertReplaceTableNames(String source) { + for (String table : srcTables) { + source = source.replaceAll("(?is)(\\s+)default\\." + table + "([\\s;\\n\\)])", "$1" + table + + "$2"); + } + return source; + } + public void filterOutput() throws IOException { String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8"); - String filteredOutput = filterSet.filter(rawOutput); + if (rewriteSourceTables) { + rawOutput = revertReplaceTableNames(rawOutput); + } + String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput)); FileUtils.writeStringToFile(outputFile, filteredOutput); } @@ -244,46 +264,17 @@ public String filter(String input) { // These are the filters which are common for every QTest. // Check specificFilterSet for QTest specific ones. - private static RegexFilterSet getFilterSet() { - // Extract the leading four digits from the unix time value. - // Use this as a prefix in order to increase the selectivity - // of the unix time stamp replacement regex. - String currentTimePrefix = Long.toString(System.currentTimeMillis()).substring(0, 4); - - String userName = System.getProperty("user.name"); - - String timePattern = "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) " - + "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " - + "\\d{2} \\d{2}:\\d{2}:\\d{2} \\w+ 20\\d{2}"; - String operatorPattern = "\"(CONDITION|COPY|DEPENDENCY_COLLECTION|DDL" - + "|EXPLAIN|FETCH|FIL|FS|FUNCTION|GBY|HASHTABLEDUMMY|HASTTABLESINK|JOIN" - + "|LATERALVIEWFORWARD|LIM|LVJ|MAP|MAPJOIN|MAPRED|MAPREDLOCAL|MOVE|OP|RS" - + "|SCR|SEL|STATS|TS|UDTF|UNION)_\\d+\""; - + private static RegexFilterSet getStaticFilterSet() { + // Pattern to remove the timestamp and other infrastructural info from the out file return new RegexFilterSet() - .addFilter("(?s)\n[^\n]*Waiting to acquire compile lock.*?Acquired the compile lock.\n", - "\n") - .addFilter(".*Acquired the compile lock.\n", "") - .addFilter("Getting log thread is interrupted, since query is done!\n", "") - .addFilter("going to print operations logs\n", "") - .addFilter("printed operations logs\n", "") - .addFilter("\\(queryId=[^\\)]*\\)", "queryId=(!!{queryId}!!)") - .addFilter("Query ID = [\\w-]+", "Query ID = !!{queryId}!!") - .addFilter("file:/\\w\\S+", "file:/!!ELIDED!!") - .addFilter("pfile:/\\w\\S+", "pfile:/!!ELIDED!!") - .addFilter("hdfs:/\\w\\S+", "hdfs:/!!ELIDED!!") - .addFilter("last_modified_by=\\w+", "last_modified_by=!!ELIDED!!") - .addFilter(timePattern, "!!TIMESTAMP!!") - .addFilter("(\\D)" + currentTimePrefix + "\\d{6}(\\D)", "$1!!UNIXTIME!!$2") - .addFilter("(\\D)" + currentTimePrefix + "\\d{9}(\\D)", "$1!!UNIXTIMEMILLIS!!$2") - .addFilter(userName, "!!{user.name}!!") - .addFilter(operatorPattern, "\"$1_!!ELIDED!!\"") - .addFilter("(?i)Time taken: [0-9\\.]* sec", "Time taken: !!ELIDED!! sec") - .addFilter(" job(:?) job_\\w+([\\s\n])", " job$1 !!{jobId}}!!$2") - .addFilter("Ended Job = job_\\w+([\\s\n])", "Ended Job = !!{jobId}!!$1") - .addFilter(".*\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}.* map = .*\n", "") - .addFilter("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\s+", "") - .addFilter("maximum memory = \\d*", "maximum memory = !!ELIDED!!"); + .addFilter("Reading log file: .*\n", "") + .addFilter("INFO : ", "") + .addFilter(".*/tmp/.*\n", MASK_PATTERN) + .addFilter(".*file:.*\n", MASK_PATTERN) + .addFilter(".*file\\..*\n", MASK_PATTERN) + .addFilter(".*CreateTime.*\n", MASK_PATTERN) + .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN) + .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN); } /** @@ -330,6 +321,12 @@ public QFile getQFile(String name) throws IOException { result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log"); result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log"); result.rewriteSourceTables = rewriteSourceTables; + result.specificFilterSet = new RegexFilterSet() + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + name + "\n", + "$1: $2: database:default\n") + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + name + "@", "$1: $2: default@") + .addFilter("name(:?) " + name + "\\.(.*)\n", "name$1 default.$2\n") + .addFilter("/" + name + ".db/", "/"); return result; } } diff --git itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java similarity index 89% rename from itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java rename to itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java index 7244bf8..f1b53f7 100644 --- itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java +++ itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java @@ -53,7 +53,6 @@ protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, public void execute(String[] commands, File resultFile) throws SQLException { beeLine.runCommands( new String[] { - "!set outputformat csv", "!record " + resultFile.getAbsolutePath() }); @@ -68,18 +67,31 @@ public void execute(String[] commands, File resultFile) throws SQLException { private void beforeExecute(QFile qFile) throws SQLException { execute( new String[] { + "!set outputformat tsv2", + "!set verbose false", + "!set silent true", + "!set showheader false", "USE default;", "SHOW TABLES;", "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;", "CREATE DATABASE `" + qFile.getName() + "`;", - "USE `" + qFile.getName() + "`;" + "USE `" + qFile.getName() + "`;", + "set hive.in.test.short.logs=true;", + "set hive.in.test.remove.logs=false;", }, qFile.getBeforeExecuteLogFile()); + beeLine.setIsTestMode(true); } private void afterExecute(QFile qFile) throws SQLException { + beeLine.setIsTestMode(false); execute( new String[] { + "set hive.in.test.short.logs=false;", + "!set verbose true", + "!set silent false", + "!set showheader true", + "!set outputformat table", "USE default;", "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;", }, diff --git itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java itests/util/src/main/java/org/apache/hive/beeline/package-info.java similarity index 100% rename from itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java rename to itests/util/src/main/java/org/apache/hive/beeline/package-info.java diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index a5c0fcd..37fffe9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -32,6 +32,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.exec.SerializationUtilities; +import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest; import org.apache.hadoop.mapreduce.MRJobConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -634,6 +635,7 @@ private static void setupChildLog4j(Configuration conf) { try { LogUtils.initHiveExecLog4j(); LogDivertAppender.registerRoutingAppender(conf); + LogDivertAppenderForTest.registerRoutingAppenderIfInTest(conf); } catch (LogInitializationException e) { System.err.println(e.getMessage()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppenderForTest.java ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppenderForTest.java new file mode 100644 index 0000000..966c264 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppenderForTest.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.log; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.RandomAccessFileAppender; +import org.apache.logging.log4j.core.appender.routing.Route; +import org.apache.logging.log4j.core.appender.routing.Routes; +import org.apache.logging.log4j.core.appender.routing.RoutingAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.config.plugins.processor.PluginEntry; +import org.apache.logging.log4j.core.config.plugins.util.PluginType; +import org.apache.logging.log4j.core.filter.AbstractFilter; +import org.apache.logging.log4j.core.layout.PatternLayout; + +/** + * Divert appender to redirect and filter test operation logs to match the output of the original + * CLI qtest results. + */ +public final class LogDivertAppenderForTest { + private LogDivertAppenderForTest() { + // Prevent instantiation + } + + /** + * A log filter that filters test messages coming from the logger. + */ + @Plugin(name = "TestFilter", category = "Core", elementType="filter", printObject = true) + private static class TestFilter extends AbstractFilter { + @Override + public Result filter(LogEvent event) { + if (event.getLevel().equals(Level.INFO) && "SessionState".equals(event.getLoggerName())) { + if (event.getMessage().getFormattedMessage().startsWith("PREHOOK:") + || event.getMessage().getFormattedMessage().startsWith("POSTHOOK:")) { + return Result.ACCEPT; + } + } + return Result.DENY; + } + + @PluginFactory + public static TestFilter createFilter() { + return new TestFilter(); + } + } + + /** + * If the HIVE_IN_TEST is set, then programmatically register a routing appender to Log4J + * configuration, which automatically writes the test log of each query to an individual file. + * The equivalent property configuration is as follows: + * # queryId based routing file appender + appender.test-query-routing.type = Routing + appender.test-query-routing.name = test-query-routing + appender.test-query-routing.routes.type = Routes + appender.test-query-routing.routes.pattern = $${ctx:queryId} + # default route + appender.test-query-routing.routes.test-route-default.type = Route + appender.test-query-routing.routes.test-route-default.key = $${ctx:queryId} + appender.test-query-routing.routes.test-route-default.app.type = NullAppender + appender.test-query-routing.routes.test-route-default.app.name = test-null-appender + # queryId based route + appender.test-query-routing.routes.test-route-mdc.type = Route + appender.test-query-routing.routes.test-route-mdc.name = test-query-routing + appender.test-query-routing.routes.test-route-mdc.app.type = RandomAccessFile + appender.test-query-routing.routes.test-route-mdc.app.name = test-query-file-appender + appender.test-query-routing.routes.test-route-mdc.app.fileName = ${sys:hive.log.dir}/${ctx:sessionId}/${ctx:queryId}.test + appender.test-query-routing.routes.test-route-mdc.app.layout.type = PatternLayout + appender.test-query-routing.routes.test-route-mdc.app.layout.pattern = %d{ISO8601} %5p %c{2}: %m%n + appender.test-query-routing.routes.test-route-mdc.app.filter.type = TestFilter + * @param conf the configuration for HiveServer2 instance + */ + public static void registerRoutingAppenderIfInTest(org.apache.hadoop.conf.Configuration conf) { + if (!conf.getBoolean(HiveConf.ConfVars.HIVE_IN_TEST.varname, + HiveConf.ConfVars.HIVE_IN_TEST.defaultBoolVal)) { + // If not in test mode, then do no create the appender + return; + } + + String logLocation = + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION); + + // Create test-null-appender to drop events without queryId + PluginEntry nullAppenderEntry = new PluginEntry(); + nullAppenderEntry.setClassName(NullAppender.class.getName()); + PluginType nullAppenderType = + new PluginType(nullAppenderEntry, NullAppender.class, "appender"); + Node nullAppenderChildNode = new Node(null, "test-null-appender", nullAppenderType); + + // Create default route where events go without queryId + PluginEntry defaultRouteEntry = new PluginEntry(); + defaultRouteEntry.setClassName(Route.class.getName()); + PluginType defaultRouteType = new PluginType(defaultRouteEntry, Route.class, ""); + Node defaultRouteNode = new Node(null, "test-route-default", defaultRouteType); + // Add the test-null-appender to the default route + defaultRouteNode.getChildren().add(nullAppenderChildNode); + + // Create queryId based route + PluginEntry queryIdRouteEntry = new PluginEntry(); + queryIdRouteEntry.setClassName(Route.class.getName()); + PluginType queryIdRouteType = new PluginType(queryIdRouteEntry, Route.class, ""); + Node queryIdRouteNode = new Node(null, "test-route-mdc", queryIdRouteType); + + // Create the queryId appender for the queryId route + PluginEntry queryIdAppenderEntry = new PluginEntry(); + queryIdAppenderEntry.setClassName(RandomAccessFileAppender.class.getName()); + PluginType queryIdAppenderType = + new PluginType(queryIdAppenderEntry, + RandomAccessFileAppender.class, "appender"); + Node queryIdAppenderNode = + new Node(queryIdRouteNode, "test-query-file-appender", queryIdAppenderType); + queryIdAppenderNode.getAttributes().put("fileName", logLocation + + "/${ctx:sessionId}/${ctx:queryId}.test"); + queryIdAppenderNode.getAttributes().put("name", "test-query-file-appender"); + // Add the queryId appender to the queryId based route + queryIdRouteNode.getChildren().add(queryIdAppenderNode); + + // Create the filter for the queryId appender + PluginEntry filterEntry = new PluginEntry(); + filterEntry.setClassName(TestFilter.class.getName()); + PluginType filterType = + new PluginType(filterEntry, TestFilter.class, ""); + Node filterNode = new Node(queryIdAppenderNode, "test-filter", filterType); + // Add the filter to the queryId appender + queryIdAppenderNode.getChildren().add(filterNode); + + // Create the layout for the queryId appender + PluginEntry layoutEntry = new PluginEntry(); + layoutEntry.setClassName(PatternLayout.class.getName()); + PluginType layoutType = + new PluginType(layoutEntry, PatternLayout.class, ""); + Node layoutNode = new Node(queryIdAppenderNode, "PatternLayout", layoutType); + layoutNode.getAttributes().put("pattern", LogDivertAppender.nonVerboseLayout); + // Add the layout to the queryId appender + queryIdAppenderNode.getChildren().add(layoutNode); + + // Create the route objects based on the Nodes + Route defaultRoute = Route.createRoute(null, "${ctx:queryId}", defaultRouteNode); + Route mdcRoute = Route.createRoute(null, null, queryIdRouteNode); + // Create the routes group + Routes routes = Routes.createRoutes("${ctx:queryId}", defaultRoute, mdcRoute); + + LoggerContext context = (LoggerContext)LogManager.getContext(false); + Configuration configuration = context.getConfiguration(); + + // Create the appender + RoutingAppender routingAppender = RoutingAppender.createAppender("test-query-routing", + "true", + routes, + configuration, + null, + null, + null); + + LoggerConfig loggerConfig = configuration.getRootLogger(); + loggerConfig.addAppender(routingAppender, null, null); + context.updateLoggers(); + routingAppender.start(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index c37a633..ded4b18 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -37,6 +37,14 @@ private final String operationName; private final LogFile logFile; + // If in test mode then the LogDivertAppenderForTest created an extra log file containing only + // the output needed for the qfile results. + private final LogFile testLogFile; + // True if we are running test and the extra test file should be used when the logs are + // requested. + private final boolean isShortLogs; + // True if the logs should be removed after the operation. Should be used only in test mode + private final boolean isRemoveLogs; private LoggingLevel opLoggingLevel = LoggingLevel.UNKNOWN; public enum LoggingLevel { @@ -51,6 +59,23 @@ public OperationLog(String name, File file, HiveConf hiveConf) throws FileNotFou String logLevel = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL); opLoggingLevel = getLoggingLevel(logLevel); } + + // If in test mod create a test log file which will contain only logs which are supposed to + // be written to the qtest output + if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) { + isRemoveLogs = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST_REMOVE_LOGS); + if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST_SHORT_LOGS)) { + testLogFile = new LogFile(new File(file.getAbsolutePath() + ".test")); + isShortLogs = true; + } else { + testLogFile = null; + isShortLogs = false; + } + } else { + testLogFile = null; + isShortLogs = false; + isRemoveLogs = true; + } } public static LoggingLevel getLoggingLevel (String mode) { @@ -79,15 +104,25 @@ public LoggingLevel getOpLoggingLevel() { * @throws java.sql.SQLException */ public List readOperationLog(boolean isFetchFirst, long maxRows) - throws SQLException{ - return logFile.read(isFetchFirst, maxRows); + throws SQLException { + if (isShortLogs) { + return testLogFile.read(isFetchFirst, maxRows); + } else { + return logFile.read(isFetchFirst, maxRows); + } } /** * Close this OperationLog when operation is closed. The log file will be removed. */ public void close() { - logFile.remove(); + if (isShortLogs) { + // In case of test, do just close the log files, do not remove them. + logFile.close(isRemoveLogs); + testLogFile.close(isRemoveLogs); + } else { + logFile.close(true); + } } /** @@ -113,12 +148,16 @@ public void close() { return readResults(maxRows); } - synchronized void remove() { + /** + * Close the logs, and remove them if specified. + * @param removeLog If true, remove the log file + */ + synchronized void close(boolean removeLog) { try { if (in != null) { in.close(); } - if (!isRemoved) { + if (!isRemoved && removeLog) { FileUtils.forceDelete(file); isRemoved = true; } @@ -135,15 +174,21 @@ private void resetIn() { } private List readResults(long nLines) throws SQLException { + List logs = new ArrayList(); if (in == null) { try { in = new BufferedReader(new InputStreamReader(new FileInputStream(file))); + // Adding name of the log file in an extra log line, so it is easier to find + // the original if there is a test error + if (isShortLogs) { + logs.add("Reading log file: " + file); + nLines--; + } } catch (FileNotFoundException e) { - return new ArrayList(); + return logs; } } - List logs = new ArrayList(); String line = ""; // if nLines <= 0, read all lines in log file. for (int i = 0; i < nLines || nLines <= 0; i++) { diff --git ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out index 2bffdf0..2c00f51 100644 --- ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out +++ ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out @@ -1,63 +1,20 @@ ->>> set hive.lock.numretries=1; -No rows affected ->>> set hive.lock.sleep.between.retries=1; -No rows affected ->>> set hive.support.concurrency=true; -No rows affected ->>> set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager; -No rows affected ->>> ->>> drop table if exists drop_with_concurrency_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): drop table if exists drop_with_concurrency_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): drop table if exists drop_with_concurrency_1 -INFO : PREHOOK: query: drop table if exists drop_with_concurrency_1 -INFO : PREHOOK: type: DROPTABLE -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: drop table if exists drop_with_concurrency_1 -INFO : POSTHOOK: type: DROPTABLE -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query drop table if exists drop_with_concurrency_1 -No rows affected ->>> create table drop_with_concurrency_1 (c1 int); -INFO : Compiling commandqueryId=(!!{queryId}!!): create table drop_with_concurrency_1 (c1 int) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table drop_with_concurrency_1 (c1 int) -INFO : PREHOOK: query: create table drop_with_concurrency_1 (c1 int) -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:drop_with_concurrency -INFO : PREHOOK: Output: drop_with_concurrency@drop_with_concurrency_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table drop_with_concurrency_1 (c1 int) -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:drop_with_concurrency -INFO : POSTHOOK: Output: drop_with_concurrency@drop_with_concurrency_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table drop_with_concurrency_1 (c1 int) -No rows affected ->>> drop table drop_with_concurrency_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): drop table drop_with_concurrency_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): drop table drop_with_concurrency_1 -INFO : PREHOOK: query: drop table drop_with_concurrency_1 -INFO : PREHOOK: type: DROPTABLE -INFO : PREHOOK: Input: drop_with_concurrency@drop_with_concurrency_1 -INFO : PREHOOK: Output: drop_with_concurrency@drop_with_concurrency_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: drop table drop_with_concurrency_1 -INFO : POSTHOOK: type: DROPTABLE -INFO : POSTHOOK: Input: drop_with_concurrency@drop_with_concurrency_1 -INFO : POSTHOOK: Output: drop_with_concurrency@drop_with_concurrency_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query drop table drop_with_concurrency_1 -No rows affected ->>> !record +PREHOOK: query: drop table if exists drop_with_concurrency_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists drop_with_concurrency_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table drop_with_concurrency_1 (c1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@drop_with_concurrency_1 +POSTHOOK: query: create table drop_with_concurrency_1 (c1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@drop_with_concurrency_1 +PREHOOK: query: drop table drop_with_concurrency_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@drop_with_concurrency_1 +PREHOOK: Output: default@drop_with_concurrency_1 +POSTHOOK: query: drop table drop_with_concurrency_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@drop_with_concurrency_1 +POSTHOOK: Output: default@drop_with_concurrency_1 diff --git ql/src/test/results/clientpositive/beeline/escape_comments.q.out ql/src/test/results/clientpositive/beeline/escape_comments.q.out index b440d9c..07fef57 100644 --- ql/src/test/results/clientpositive/beeline/escape_comments.q.out +++ ql/src/test/results/clientpositive/beeline/escape_comments.q.out @@ -1,406 +1,217 @@ ->>> create database escape_comments_db comment 'a\nb'; -INFO : Compiling commandqueryId=(!!{queryId}!!): create database escape_comments_db comment 'a\nb' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create database escape_comments_db comment 'a\nb' -INFO : PREHOOK: query: create database escape_comments_db comment 'a\nb' -INFO : PREHOOK: type: CREATEDATABASE -INFO : PREHOOK: Output: database:escape_comments_db -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create database escape_comments_db comment 'a\nb' -INFO : POSTHOOK: type: CREATEDATABASE -INFO : POSTHOOK: Output: database:escape_comments_db -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create database escape_comments_db comment 'a\nb' -No rows affected ->>> use escape_comments_db; -INFO : Compiling commandqueryId=(!!{queryId}!!): use escape_comments_db -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): use escape_comments_db -INFO : PREHOOK: query: use escape_comments_db -INFO : PREHOOK: type: SWITCHDATABASE -INFO : PREHOOK: Input: database:escape_comments_db -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: use escape_comments_db -INFO : POSTHOOK: type: SWITCHDATABASE -INFO : POSTHOOK: Input: database:escape_comments_db -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query use escape_comments_db -No rows affected ->>> create table escape_comments_tbl1 -(col1 string comment 'a\nb\'\;') comment 'a\nb' -partitioned by (p1 string comment 'a\nb'); -INFO : Compiling commandqueryId=(!!{queryId}!!): create table escape_comments_tbl1 -(col1 string comment 'a\nb\'\;') comment 'a\nb' -partitioned by (p1 string comment 'a\nb') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table escape_comments_tbl1 -(col1 string comment 'a\nb\'\;') comment 'a\nb' -partitioned by (p1 string comment 'a\nb') -INFO : PREHOOK: query: create table escape_comments_tbl1 +PREHOOK: query: create database escape_comments_db comment 'a\nb' +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:escape_comments_db +POSTHOOK: query: create database escape_comments_db comment 'a\nb' +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:escape_comments_db +PREHOOK: query: use escape_comments_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:escape_comments_db +POSTHOOK: query: use escape_comments_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:escape_comments_db +PREHOOK: query: create table escape_comments_tbl1 (col1 string comment 'a\nb\'\;') comment 'a\nb' partitioned by (p1 string comment 'a\nb') -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:escape_comments_db -INFO : PREHOOK: Output: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table escape_comments_tbl1 +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:escape_comments_db +PREHOOK: Output: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: create table escape_comments_tbl1 (col1 string comment 'a\nb\'\;') comment 'a\nb' partitioned by (p1 string comment 'a\nb') -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:escape_comments_db -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_tbl1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table escape_comments_tbl1 -(col1 string comment 'a\nb\'\;') comment 'a\nb' -partitioned by (p1 string comment 'a\nb') -No rows affected ->>> create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' -as select col1 from escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' -as select col1 from escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:escape_comments_db +POSTHOOK: Output: escape_comments_db@escape_comments_tbl1 +PREHOOK: query: create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' as select col1 from escape_comments_tbl1 -INFO : PREHOOK: query: create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' +PREHOOK: type: CREATEVIEW +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +PREHOOK: Output: database:escape_comments_db +PREHOOK: Output: escape_comments_db@escape_comments_view1 +POSTHOOK: query: create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' as select col1 from escape_comments_tbl1 -INFO : PREHOOK: type: CREATEVIEW -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : PREHOOK: Output: database:escape_comments_db -INFO : PREHOOK: Output: escape_comments_db@escape_comments_view1 -INFO : Starting task [Stage-1:DDL] in serial mode -INFO : POSTHOOK: query: create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' -as select col1 from escape_comments_tbl1 -INFO : POSTHOOK: type: CREATEVIEW -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : POSTHOOK: Output: database:escape_comments_db -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_view1 -INFO : POSTHOOK: Lineage: escape_comments_view1.col1 SIMPLE [(escape_comments_tbl1)escape_comments_tbl1.FieldSchema(name:col1, type:string, comment:a +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: Output: database:escape_comments_db +POSTHOOK: Output: escape_comments_db@escape_comments_view1 +POSTHOOK: Lineage: escape_comments_view1.col1 SIMPLE [(escape_comments_tbl1)escape_comments_tbl1.FieldSchema(name:col1, type:string, comment:a b';), ] -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' -as select col1 from escape_comments_tbl1 -No rows affected ->>> create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb'; -INFO : Compiling commandqueryId=(!!{queryId}!!): create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' -INFO : PREHOOK: query: create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' -INFO : PREHOOK: type: CREATEINDEX -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' -INFO : POSTHOOK: type: CREATEINDEX -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' -No rows affected ->>> ->>> describe database extended escape_comments_db; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe database extended escape_comments_db -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:db_name, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer), FieldSchema(name:location, type:string, comment:from deserializer), FieldSchema(name:owner_name, type:string, comment:from deserializer), FieldSchema(name:owner_type, type:string, comment:from deserializer), FieldSchema(name:parameters, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe database extended escape_comments_db -INFO : PREHOOK: query: describe database extended escape_comments_db -INFO : PREHOOK: type: DESCDATABASE -INFO : PREHOOK: Input: database:escape_comments_db -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe database extended escape_comments_db -INFO : POSTHOOK: type: DESCDATABASE -INFO : POSTHOOK: Input: database:escape_comments_db -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe database extended escape_comments_db -'db_name','comment','location','owner_name','owner_type','parameters' -'escape_comments_db','a\nb','location/in/test','user','USER','' -1 row selected ->>> describe database escape_comments_db; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe database escape_comments_db -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:db_name, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer), FieldSchema(name:location, type:string, comment:from deserializer), FieldSchema(name:owner_name, type:string, comment:from deserializer), FieldSchema(name:owner_type, type:string, comment:from deserializer), FieldSchema(name:parameters, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe database escape_comments_db -INFO : PREHOOK: query: describe database escape_comments_db -INFO : PREHOOK: type: DESCDATABASE -INFO : PREHOOK: Input: database:escape_comments_db -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe database escape_comments_db -INFO : POSTHOOK: type: DESCDATABASE -INFO : POSTHOOK: Input: database:escape_comments_db -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe database escape_comments_db -'db_name','comment','location','owner_name','owner_type','parameters' -'escape_comments_db','a\nb','location/in/test','user','USER','' -1 row selected ->>> show create table escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): show create table escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:createtab_stmt, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): show create table escape_comments_tbl1 -INFO : PREHOOK: query: show create table escape_comments_tbl1 -INFO : PREHOOK: type: SHOW_CREATETABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: show create table escape_comments_tbl1 -INFO : POSTHOOK: type: SHOW_CREATETABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query show create table escape_comments_tbl1 -'createtab_stmt' -'CREATE TABLE `escape_comments_tbl1`(' -' `col1` string COMMENT 'a\nb\'\;')' -'COMMENT 'a\nb'' -'PARTITIONED BY ( ' -' `p1` string COMMENT 'a\nb')' -'ROW FORMAT SERDE ' -' 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' ' -'STORED AS INPUTFORMAT ' -' 'org.apache.hadoop.mapred.TextInputFormat' ' -'OUTPUTFORMAT ' -' 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'' -'LOCATION' -' 'file:/!!ELIDED!! -'TBLPROPERTIES (' -' 'transient_lastDdlTime'='!!UNIXTIME!!')' -15 rows selected ->>> describe formatted escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe formatted escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe formatted escape_comments_tbl1 -INFO : PREHOOK: query: describe formatted escape_comments_tbl1 -INFO : PREHOOK: type: DESCTABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe formatted escape_comments_tbl1 -INFO : POSTHOOK: type: DESCTABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe formatted escape_comments_tbl1 -'col_name','data_type','comment' -'# col_name ','data_type ','comment ' -'','NULL','NULL' -'col1','string','a\nb';' -'','NULL','NULL' -'# Partition Information','NULL','NULL' -'# col_name ','data_type ','comment ' -'','NULL','NULL' -'p1','string','a\nb' -'','NULL','NULL' -'# Detailed Table Information','NULL','NULL' -'Database: ','escape_comments_db ','NULL' -'Owner: ','user ','NULL' -'CreateTime: ','!!TIMESTAMP!!','NULL' -'LastAccessTime: ','UNKNOWN ','NULL' -'Retention: ','0 ','NULL' -'Location: ','file:/!!ELIDED!! -'Table Type: ','MANAGED_TABLE ','NULL' -'Table Parameters:','NULL','NULL' -'','COLUMN_STATS_ACCURATE','{\"BASIC_STATS\":\"true\"}' -'','comment ','a\nb ' -'','numFiles ','0 ' -'','numPartitions ','0 ' -'','numRows ','0 ' -'','rawDataSize ','0 ' -'','totalSize ','0 ' -'','transient_lastDdlTime','!!UNIXTIME!! ' -'','NULL','NULL' -'# Storage Information','NULL','NULL' -'SerDe Library: ','org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe','NULL' -'InputFormat: ','org.apache.hadoop.mapred.TextInputFormat','NULL' -'OutputFormat: ','org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat','NULL' -'Compressed: ','No ','NULL' -'Num Buckets: ','-1 ','NULL' -'Bucket Columns: ','[] ','NULL' -'Sort Columns: ','[] ','NULL' -'Storage Desc Params:','NULL','NULL' -'','serialization.format','1 ' -37 rows selected ->>> describe pretty escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe pretty escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe pretty escape_comments_tbl1 -INFO : PREHOOK: query: describe pretty escape_comments_tbl1 -INFO : PREHOOK: type: DESCTABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe pretty escape_comments_tbl1 -INFO : POSTHOOK: type: DESCTABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe pretty escape_comments_tbl1 -'col_name','data_type','comment' -'col_name ','data_type ','comment' -'','','' -'col1 ','string ','a' -' ',' ','b';' -'p1 ','string ','a' -' ',' ','b' -'','NULL','NULL' -'# Partition Information','NULL','NULL' -'col_name ','data_type ','comment' -'','','' -'p1 ','string ','a' -' ',' ','b' -12 rows selected ->>> describe escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe escape_comments_tbl1 -INFO : PREHOOK: query: describe escape_comments_tbl1 -INFO : PREHOOK: type: DESCTABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe escape_comments_tbl1 -INFO : POSTHOOK: type: DESCTABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe escape_comments_tbl1 -'col_name','data_type','comment' -'col1','string','a\nb';' -'p1','string','a\nb' -'','NULL','NULL' -'# Partition Information','NULL','NULL' -'# col_name ','data_type ','comment ' -'','NULL','NULL' -'p1','string','a\nb' -7 rows selected ->>> show create table escape_comments_view1; -INFO : Compiling commandqueryId=(!!{queryId}!!): show create table escape_comments_view1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:createtab_stmt, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): show create table escape_comments_view1 -INFO : PREHOOK: query: show create table escape_comments_view1 -INFO : PREHOOK: type: SHOW_CREATETABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_view1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: show create table escape_comments_view1 -INFO : POSTHOOK: type: SHOW_CREATETABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_view1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query show create table escape_comments_view1 -'createtab_stmt' -'CREATE VIEW `escape_comments_view1` AS SELECT `col1` AS `col1` FROM (select `escape_comments_tbl1`.`col1` from `escape_comments_db`.`escape_comments_tbl1`) `escape_comments_db.escape_comments_view1`' -1 row selected ->>> describe formatted escape_comments_view1; -INFO : Compiling commandqueryId=(!!{queryId}!!): describe formatted escape_comments_view1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): describe formatted escape_comments_view1 -INFO : PREHOOK: query: describe formatted escape_comments_view1 -INFO : PREHOOK: type: DESCTABLE -INFO : PREHOOK: Input: escape_comments_db@escape_comments_view1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: describe formatted escape_comments_view1 -INFO : POSTHOOK: type: DESCTABLE -INFO : POSTHOOK: Input: escape_comments_db@escape_comments_view1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query describe formatted escape_comments_view1 -'col_name','data_type','comment' -'# col_name ','data_type ','comment ' -'','NULL','NULL' -'col1','string','a\nb' -'','NULL','NULL' -'# Detailed Table Information','NULL','NULL' -'Database: ','escape_comments_db ','NULL' -'Owner: ','user ','NULL' -'CreateTime: ','!!TIMESTAMP!!','NULL' -'LastAccessTime: ','UNKNOWN ','NULL' -'Retention: ','0 ','NULL' -'Table Type: ','VIRTUAL_VIEW ','NULL' -'Table Parameters:','NULL','NULL' -'','comment ','a\nb ' -'','transient_lastDdlTime','!!UNIXTIME!! ' -'','NULL','NULL' -'# Storage Information','NULL','NULL' -'SerDe Library: ','null ','NULL' -'InputFormat: ','org.apache.hadoop.mapred.TextInputFormat','NULL' -'OutputFormat: ','org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat','NULL' -'Compressed: ','No ','NULL' -'Num Buckets: ','-1 ','NULL' -'Bucket Columns: ','[] ','NULL' -'Sort Columns: ','[] ','NULL' -'','NULL','NULL' -'# View Information','NULL','NULL' -'View Original Text: ','select col1 from escape_comments_tbl1','NULL' -'View Expanded Text: ','SELECT `col1` AS `col1` FROM (select `escape_comments_tbl1`.`col1` from `escape_comments_db`.`escape_comments_tbl1`) `escape_comments_db.escape_comments_view1`','NULL' -'View Rewrite Enabled:','No ','NULL' -28 rows selected ->>> show formatted index on escape_comments_tbl1; -INFO : Compiling commandqueryId=(!!{queryId}!!): show formatted index on escape_comments_tbl1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:idx_name, type:string, comment:from deserializer), FieldSchema(name:tab_name, type:string, comment:from deserializer), FieldSchema(name:col_names, type:string, comment:from deserializer), FieldSchema(name:idx_tab_name, type:string, comment:from deserializer), FieldSchema(name:idx_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): show formatted index on escape_comments_tbl1 -INFO : PREHOOK: query: show formatted index on escape_comments_tbl1 -INFO : PREHOOK: type: SHOWINDEXES -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: show formatted index on escape_comments_tbl1 -INFO : POSTHOOK: type: SHOWINDEXES -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query show formatted index on escape_comments_tbl1 -'idx_name','tab_name','col_names','idx_tab_name','idx_type','comment' -'idx_name ','tab_name ','col_names ','idx_tab_name ','idx_type ','comment ' -'','NULL','NULL','NULL','NULL','NULL' -'','NULL','NULL','NULL','NULL','NULL' -'index2 ','escape_comments_tbl1','col1 ','escape_comments_db__escape_comments_tbl1_index2__','compact ','a\nb ' -4 rows selected ->>> ->>> drop database escape_comments_db cascade; -INFO : Compiling commandqueryId=(!!{queryId}!!): drop database escape_comments_db cascade -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): drop database escape_comments_db cascade -INFO : PREHOOK: query: drop database escape_comments_db cascade -INFO : PREHOOK: type: DROPDATABASE -INFO : PREHOOK: Input: database:escape_comments_db -INFO : PREHOOK: Output: database:escape_comments_db -INFO : PREHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ -INFO : PREHOOK: Output: escape_comments_db@escape_comments_tbl1 -INFO : PREHOOK: Output: escape_comments_db@escape_comments_view1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: drop database escape_comments_db cascade -INFO : POSTHOOK: type: DROPDATABASE -INFO : POSTHOOK: Input: database:escape_comments_db -INFO : POSTHOOK: Output: database:escape_comments_db -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_tbl1 -INFO : POSTHOOK: Output: escape_comments_db@escape_comments_view1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query drop database escape_comments_db cascade -No rows affected ->>> !record +PREHOOK: query: create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' +PREHOOK: type: CREATEINDEX +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ +PREHOOK: query: describe database extended escape_comments_db +PREHOOK: type: DESCDATABASE +PREHOOK: Input: database:escape_comments_db +POSTHOOK: query: describe database extended escape_comments_db +POSTHOOK: type: DESCDATABASE +POSTHOOK: Input: database:escape_comments_db +escape_comments_db a\nb location/in/test user USER +PREHOOK: query: describe database escape_comments_db +PREHOOK: type: DESCDATABASE +PREHOOK: Input: database:escape_comments_db +POSTHOOK: query: describe database escape_comments_db +POSTHOOK: type: DESCDATABASE +POSTHOOK: Input: database:escape_comments_db +escape_comments_db a\nb location/in/test user USER +PREHOOK: query: show create table escape_comments_tbl1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: show create table escape_comments_tbl1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +CREATE TABLE `escape_comments_tbl1`( + `col1` string COMMENT 'a\nb\'\;') +COMMENT 'a\nb' +PARTITIONED BY ( + `p1` string COMMENT 'a\nb') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( +#### A masked pattern was here #### +PREHOOK: query: describe formatted escape_comments_tbl1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: describe formatted escape_comments_tbl1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +# col_name data_type comment + NULL NULL +col1 string a\nb'; + NULL NULL +# Partition Information NULL NULL +# col_name data_type comment + NULL NULL +p1 string a\nb + NULL NULL +# Detailed Table Information NULL NULL +Database: escape_comments_db NULL +Owner: user NULL +#### A masked pattern was here #### +LastAccessTime: UNKNOWN NULL +Retention: 0 NULL +#### A masked pattern was here #### +Table Type: MANAGED_TABLE NULL +Table Parameters: NULL NULL + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + comment a\nb + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + NULL NULL +# Storage Information NULL NULL +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe NULL +InputFormat: org.apache.hadoop.mapred.TextInputFormat NULL +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat NULL +Compressed: No NULL +Num Buckets: -1 NULL +Bucket Columns: [] NULL +Sort Columns: [] NULL +Storage Desc Params: NULL NULL + serialization.format 1 +PREHOOK: query: describe pretty escape_comments_tbl1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: describe pretty escape_comments_tbl1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +col_name data_type comment + +col1 string a + b'; +p1 string a + b + NULL NULL +# Partition Information NULL NULL +col_name data_type comment + +p1 string a + b +PREHOOK: query: describe escape_comments_tbl1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: escape_comments_db@escape_comments_tbl1 +POSTHOOK: query: describe escape_comments_tbl1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: escape_comments_db@escape_comments_tbl1 +col1 string a\nb'; +p1 string a\nb + NULL NULL +# Partition Information NULL NULL +# col_name data_type comment + NULL NULL +p1 string a\nb +PREHOOK: query: show create table escape_comments_view1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: escape_comments_db@escape_comments_view1 +POSTHOOK: query: show create table escape_comments_view1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: escape_comments_db@escape_comments_view1 +CREATE VIEW `escape_comments_view1` AS SELECT `col1` AS `col1` FROM (select `escape_comments_tbl1`.`col1` from `escape_comments_db`.`escape_comments_tbl1`) `escape_comments_db.escape_comments_view1` +PREHOOK: query: describe formatted escape_comments_view1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: escape_comments_db@escape_comments_view1 +POSTHOOK: query: describe formatted escape_comments_view1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: escape_comments_db@escape_comments_view1 +# col_name data_type comment + NULL NULL +col1 string a\nb + NULL NULL +# Detailed Table Information NULL NULL +Database: escape_comments_db NULL +Owner: user NULL +#### A masked pattern was here #### +LastAccessTime: UNKNOWN NULL +Retention: 0 NULL +Table Type: VIRTUAL_VIEW NULL +Table Parameters: NULL NULL + comment a\nb +#### A masked pattern was here #### + NULL NULL +# Storage Information NULL NULL +SerDe Library: null NULL +InputFormat: org.apache.hadoop.mapred.TextInputFormat NULL +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat NULL +Compressed: No NULL +Num Buckets: -1 NULL +Bucket Columns: [] NULL +Sort Columns: [] NULL + NULL NULL +# View Information NULL NULL +View Original Text: select col1 from escape_comments_tbl1 NULL +View Expanded Text: SELECT `col1` AS `col1` FROM (select `escape_comments_tbl1`.`col1` from `escape_comments_db`.`escape_comments_tbl1`) `escape_comments_db.escape_comments_view1` NULL +View Rewrite Enabled: No NULL +PREHOOK: query: show formatted index on escape_comments_tbl1 +PREHOOK: type: SHOWINDEXES +POSTHOOK: query: show formatted index on escape_comments_tbl1 +POSTHOOK: type: SHOWINDEXES +idx_name tab_name col_names idx_tab_name idx_type comment + NULL NULL NULL NULL NULL + NULL NULL NULL NULL NULL +index2 escape_comments_tbl1 col1 escape_comments_db__escape_comments_tbl1_index2__ compact a\nb +PREHOOK: query: drop database escape_comments_db cascade +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:escape_comments_db +PREHOOK: Output: database:escape_comments_db +PREHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ +PREHOOK: Output: escape_comments_db@escape_comments_tbl1 +PREHOOK: Output: escape_comments_db@escape_comments_view1 +POSTHOOK: query: drop database escape_comments_db cascade +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:escape_comments_db +POSTHOOK: Output: database:escape_comments_db +POSTHOOK: Output: escape_comments_db@escape_comments_db__escape_comments_tbl1_index2__ +POSTHOOK: Output: escape_comments_db@escape_comments_tbl1 +POSTHOOK: Output: escape_comments_db@escape_comments_view1 diff --git ql/src/test/results/clientpositive/beeline/select_dummy_source.q.out ql/src/test/results/clientpositive/beeline/select_dummy_source.q.out new file mode 100644 index 0000000..b6fa03f --- /dev/null +++ ql/src/test/results/clientpositive/beeline/select_dummy_source.q.out @@ -0,0 +1,251 @@ +PREHOOK: query: explain +select 'a', 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select 'a', 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'a' (type: string), 100 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select 'a', 100 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'a', 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +a 100 +PREHOOK: query: explain +select 1 + 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select 1 + 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select 1 + 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 1 + 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +2 +PREHOOK: query: explain +select explode(array('a', 'b')) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select explode(array('a', 'b')) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array('a','b') (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + function name: explode + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select explode(array('a', 'b')) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select explode(array('a', 'b')) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +a +b +PREHOOK: query: explain +select 'a', 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select 'a', 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'a' (type: string), 100 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select 'a', 100 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'a', 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +a 100 +PREHOOK: query: explain +select 1 + 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select 1 + 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select 1 + 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 1 + 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +2 +PREHOOK: query: explain +select explode(array('a', 'b')) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select explode(array('a', 'b')) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array('a','b') (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + function name: explode + ListSink + +PREHOOK: query: select explode(array('a', 'b')) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select explode(array('a', 'b')) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +a +b +PREHOOK: query: explain +select 2 + 3,x from (select 1 + 2 x) X +PREHOOK: type: QUERY +POSTHOOK: query: explain +select 2 + 3,x from (select 1 + 2 x) X +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 5 (type: int), 3 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select 2 + 3,x from (select 1 + 2 x) X +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 2 + 3,x from (select 1 + 2 x) X +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +5 3 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out index 70a37ca..c943b03 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out @@ -1,948 +1,490 @@ ->>> set hive.strict.checks.bucketing=false; -No rows affected ->>> ->>> ->>> ->>> ->>> ->>> create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_1 -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_1 -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_1 -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_1 -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_1 -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_1 -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> ->>> load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_1.smb_bucket_1 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_2 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_1.smb_bucket_2 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_1@smb_bucket_3 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_1.smb_bucket_3 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_1@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -No rows affected ->>> ->>> set hive.cbo.enable=false; -No rows affected ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> -- SORT_QUERY_RESULTS ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -No rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','NULL','NULL' -'5','val_5','NULL','NULL' -'10','val_10','NULL','NULL' -5 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 NULL NULL +5 val_5 NULL NULL +10 val_10 NULL NULL +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -'NULL','NULL','25','val_25' -'NULL','NULL','30','val_30' -4 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +NULL NULL 20 val_20 +NULL NULL 23 val_23 +NULL NULL 25 val_25 +NULL NULL 30 val_30 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','NULL','NULL' -'5','val_5','NULL','NULL' -'10','val_10','NULL','NULL' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -'NULL','NULL','25','val_25' -'NULL','NULL','30','val_30' -9 rows selected ->>> ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 NULL NULL +5 val_5 NULL NULL +10 val_10 NULL NULL +NULL NULL 20 val_20 +NULL NULL 23 val_23 +NULL NULL 25 val_25 +NULL NULL 30 val_30 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -No rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','NULL','NULL' -'5','val_5','NULL','NULL' -'10','val_10','NULL','NULL' -5 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 NULL NULL +5 val_5 NULL NULL +10 val_10 NULL NULL +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -'NULL','NULL','25','val_25' -'NULL','NULL','30','val_30' -4 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +NULL NULL 20 val_20 +NULL NULL 23 val_23 +NULL NULL 25 val_25 +NULL NULL 30 val_30 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_1@smb_bucket_2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','NULL','NULL' -'5','val_5','NULL','NULL' -'10','val_10','NULL','NULL' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -'NULL','NULL','25','val_25' -'NULL','NULL','30','val_30' -9 rows selected ->>> ->>> ->>> ->>> ->>> ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_2 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 NULL NULL +5 val_5 NULL NULL +10 val_10 NULL NULL +NULL NULL 20 val_20 +NULL NULL 23 val_23 +NULL NULL 25 val_25 +NULL NULL 30 val_30 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_10.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_10.q.out index bdfaefb..50706f4 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_10.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_10.q.out @@ -1,248 +1,107 @@ ->>> set hive.strict.checks.bucketing=false; -No rows affected ->>> ->>> ->>> create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_10 -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_10 -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE -No rows affected ->>> ->>> alter table tmp_smb_bucket_10 add partition (ds = '1'); -INFO : Compiling commandqueryId=(!!{queryId}!!): alter table tmp_smb_bucket_10 add partition (ds = '1') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): alter table tmp_smb_bucket_10 add partition (ds = '1') -INFO : PREHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '1') -INFO : PREHOOK: type: ALTERTABLE_ADDPARTS -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '1') -INFO : POSTHOOK: type: ALTERTABLE_ADDPARTS -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query alter table tmp_smb_bucket_10 add partition (ds = '1') -No rows affected ->>> alter table tmp_smb_bucket_10 add partition (ds = '2'); -INFO : Compiling commandqueryId=(!!{queryId}!!): alter table tmp_smb_bucket_10 add partition (ds = '2') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): alter table tmp_smb_bucket_10 add partition (ds = '2') -INFO : PREHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '2') -INFO : PREHOOK: type: ALTERTABLE_ADDPARTS -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '2') -INFO : POSTHOOK: type: ALTERTABLE_ADDPARTS -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10 -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query alter table tmp_smb_bucket_10 add partition (ds = '2') -No rows affected ->>> ->>> -- add dummy files to make sure that the number of files in each partition is same as number of buckets ->>> ->>> load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1'); -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_10.tmp_smb_bucket_10 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -No rows affected ->>> load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1'); -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_10.tmp_smb_bucket_10 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') -No rows affected ->>> ->>> load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2'); -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=2 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_10.tmp_smb_bucket_10 partition (ds=2) from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -No rows affected ->>> load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2'); -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=2 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_10.tmp_smb_bucket_10 partition (ds=2) from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_10@tmp_smb_bucket_10@ds=2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> explain -select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b -on (a.ds = '1' and b.ds = '2' and - a.userid = b.userid and - a.pageid = b.pageid and - a.postid = b.postid and - a.type = b.type); -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b -on (a.ds = '1' and b.ds = '2' and - a.userid = b.userid and - a.pageid = b.pageid and - a.postid = b.postid and - a.type = b.type) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b -on (a.ds = '1' and b.ds = '2' and - a.userid = b.userid and - a.pageid = b.pageid and - a.postid = b.postid and - a.type = b.type) -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b -on (a.ds = '1' and b.ds = '2' and - a.userid = b.userid and - a.pageid = b.pageid and - a.postid = b.postid and - a.type = b.type) -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +PREHOOK: query: create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_smb_bucket_10 +POSTHOOK: query: create table tmp_smb_bucket_10(userid int, pageid int, postid int, type string) partitioned by (ds string) CLUSTERED BY (userid) SORTED BY (pageid, postid, type, userid) INTO 2 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_smb_bucket_10 +PREHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '1') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@tmp_smb_bucket_10 +POSTHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '1') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@tmp_smb_bucket_10 +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=1 +PREHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '2') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@tmp_smb_bucket_10 +POSTHOOK: query: alter table tmp_smb_bucket_10 add partition (ds = '2') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@tmp_smb_bucket_10 +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=2 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@tmp_smb_bucket_10@ds=1 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=1 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@tmp_smb_bucket_10@ds=1 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='1') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=1 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@tmp_smb_bucket_10@ds=2 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=2 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@tmp_smb_bucket_10@ds=2 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' INTO TABLE tmp_smb_bucket_10 partition(ds='2') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@tmp_smb_bucket_10@ds=2 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b on (a.ds = '1' and b.ds = '2' and a.userid = b.userid and a.pageid = b.pageid and a.postid = b.postid and a.type = b.type) -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from tmp_smb_bucket_10 a join tmp_smb_bucket_10 b on (a.ds = '1' and b.ds = '2' and a.userid = b.userid and a.pageid = b.pageid and a.postid = b.postid and a.type = b.type) -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 3 Data size: 414 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: (userid is not null and pageid is not null and postid is not null and type is not null) (type: boolean)' -' Statistics: Num rows: 3 Data size: 414 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 userid (type: int), pageid (type: int), postid (type: int), type (type: string)' -' 1 userid (type: int), pageid (type: int), postid (type: int), type (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3, _col8, _col9, _col10, _col11' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), '1' (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), '2' (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 414 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (userid is not null and pageid is not null and postid is not null and type is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 414 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 userid (type: int), pageid (type: int), postid (type: int), type (type: string) + 1 userid (type: int), pageid (type: int), postid (type: int), type (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col8, _col9, _col10, _col11 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), '1' (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), '2' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out index 19c07a0..b53e670 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out @@ -1,2563 +1,2161 @@ ->>> set hive.mapred.mode=nonstrict; -No rows affected ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> set hive.cbo.enable=false; -No rows affected ->>> ->>> set hive.exec.reducers.max = 1; -No rows affected ->>> set hive.merge.mapfiles=false; -No rows affected ->>> set hive.merge.mapredfiles=false; -No rows affected ->>> ->>> -- This test verifies that the output of a sort merge join on 2 partitions (one on each side of the join) is bucketed ->>> ->>> -- Create two bucketed and sorted tables ->>> CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_11 -INFO : PREHOOK: Output: smb_mapjoin_11@test_table1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_11 -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -No rows affected ->>> CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_11 -INFO : PREHOOK: Output: smb_mapjoin_11@test_table2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_11 -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -No rows affected ->>> ->>> FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT *; -INFO : Compiling commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_col0, type:int, comment:null), FieldSchema(name:_col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INFO : PREHOOK: query: FROM default.src +PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table1 +POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table1 +PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table2 +POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table2 +PREHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: default@src -INFO : PREHOOK: Output: smb_mapjoin_11@test_table1@ds=1 -INFO : PREHOOK: Output: smb_mapjoin_11@test_table2@ds=1 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 2 -INFO : Launching Job 1 out of 2 -INFO : Starting task [Stage-2:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_11.test_table1 partition (ds=1) from file:/!!ELIDED!! -INFO : Launching Job 2 out of 2 -INFO : Starting task [Stage-4:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-3:STATS] in serial mode -INFO : Starting task [Stage-1:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_11.test_table2 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-5:STATS] in serial mode -INFO : POSTHOOK: query: FROM default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_table1@ds=1 +PREHOOK: Output: default@test_table2@ds=1 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: default@src -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table1@ds=1 -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table2@ds=1 -INFO : POSTHOOK: Lineage: test_table1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-2: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-4: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -No rows affected ->>> ->>> ->>> ->>> ->>> -- Create a bucketed table ->>> CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_11 -INFO : PREHOOK: Output: smb_mapjoin_11@test_table3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_11 -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS -No rows affected ->>> ->>> -- Insert data into the bucketed table by joining the two bucketed and sorted tables, bucketing is not enforced ->>> EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : PREHOOK: query: EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-4:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_table1@ds=1 +POSTHOOK: Output: default@test_table2@ds=1 +POSTHOOK: Lineage: test_table1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table3 +POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table3 +PREHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -' Stage-2 depends on stages: Stage-0' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col7' -' Position of Big Table: 0' -' BucketMapJoin: true' -' Select Operator' -' expressions: _col0 (type: int), _col7 (type: string)' -' outputColumnNames: _col0, _col1' -' Reduce Output Operator' -' null sort order: ' -' sort order: ' -' Map-reduce partition columns: _col0 (type: int)' -' tag: -1' -' value expressions: _col0 (type: int), _col1 (type: string)' -' auto parallelism: false' -' Path -> Alias:' -' file:/!!ELIDED!! [a]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: ds=1' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' partition values:' -' ds 1' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table1' -' numFiles 16' -' numRows 500' -' partition_columns ds' -' partition_columns.types string' -' rawDataSize 5312' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table1' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_11.test_table1' -' name: smb_mapjoin_11.test_table1' -' Truncated Path -> Alias:' -' /smb_mapjoin_11.db/test_table1/ds=1 [a]' -' Needs Tagging: false' -' Reduce Operator Tree:' -' Select Operator' -' expressions: VALUE._col0 (type: int), VALUE._col1 (type: string)' -' outputColumnNames: _col0, _col1' -' File Output Operator' -' compressed: false' -' GlobalTableId: 1' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 16' -' Static Partition Specification: ds=1/' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_11.test_table3' -' TotalFiles: 16' -' GatherStats: true' -' MultiFileSpray: true' -'' -' Stage: Stage-0' -' Move Operator' -' tables:' -' partition:' -' ds 1' -' replace: true' -' source: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_11.test_table3' -'' -' Stage: Stage-2' -' Stats-Aggr Operator' -' Stats Aggregation Key Prefix: file:/!!ELIDED!! -'' -167 rows selected ->>> ->>> INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table2 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table2@ds=1 -INFO : PREHOOK: Output: smb_mapjoin_11@test_table3@ds=1 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_11.test_table3 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table2 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table2@ds=1 -INFO : POSTHOOK: Output: smb_mapjoin_11@test_table3@ds=1 -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' -No rows affected ->>> ->>> SELECT * FROM test_table1 ORDER BY key; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT * FROM test_table1 ORDER BY key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:test_table1.key, type:int, comment:null), FieldSchema(name:test_table1.value, type:string, comment:null), FieldSchema(name:test_table1.ds, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT * FROM test_table1 ORDER BY key -INFO : PREHOOK: query: SELECT * FROM test_table1 ORDER BY key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT * FROM test_table1 ORDER BY key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT * FROM test_table1 ORDER BY key -'test_table1.key','test_table1.value','test_table1.ds' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'2','val_2','1' -'4','val_4','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'8','val_8','1' -'9','val_9','1' -'10','val_10','1' -'11','val_11','1' -'12','val_12','1' -'12','val_12','1' -'15','val_15','1' -'15','val_15','1' -'17','val_17','1' -'18','val_18','1' -'18','val_18','1' -'19','val_19','1' -'20','val_20','1' -'24','val_24','1' -'24','val_24','1' -'26','val_26','1' -'26','val_26','1' -'27','val_27','1' -'28','val_28','1' -'30','val_30','1' -'33','val_33','1' -'34','val_34','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'37','val_37','1' -'37','val_37','1' -'41','val_41','1' -'42','val_42','1' -'42','val_42','1' -'43','val_43','1' -'44','val_44','1' -'47','val_47','1' -'51','val_51','1' -'51','val_51','1' -'53','val_53','1' -'54','val_54','1' -'57','val_57','1' -'58','val_58','1' -'58','val_58','1' -'64','val_64','1' -'65','val_65','1' -'66','val_66','1' -'67','val_67','1' -'67','val_67','1' -'69','val_69','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'72','val_72','1' -'72','val_72','1' -'74','val_74','1' -'76','val_76','1' -'76','val_76','1' -'77','val_77','1' -'78','val_78','1' -'80','val_80','1' -'82','val_82','1' -'83','val_83','1' -'83','val_83','1' -'84','val_84','1' -'84','val_84','1' -'85','val_85','1' -'86','val_86','1' -'87','val_87','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'92','val_92','1' -'95','val_95','1' -'95','val_95','1' -'96','val_96','1' -'97','val_97','1' -'97','val_97','1' -'98','val_98','1' -'98','val_98','1' -'100','val_100','1' -'100','val_100','1' -'103','val_103','1' -'103','val_103','1' -'104','val_104','1' -'104','val_104','1' -'105','val_105','1' -'111','val_111','1' -'113','val_113','1' -'113','val_113','1' -'114','val_114','1' -'116','val_116','1' -'118','val_118','1' -'118','val_118','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'120','val_120','1' -'120','val_120','1' -'125','val_125','1' -'125','val_125','1' -'126','val_126','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'129','val_129','1' -'129','val_129','1' -'131','val_131','1' -'133','val_133','1' -'134','val_134','1' -'134','val_134','1' -'136','val_136','1' -'137','val_137','1' -'137','val_137','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'143','val_143','1' -'145','val_145','1' -'146','val_146','1' -'146','val_146','1' -'149','val_149','1' -'149','val_149','1' -'150','val_150','1' -'152','val_152','1' -'152','val_152','1' -'153','val_153','1' -'155','val_155','1' -'156','val_156','1' -'157','val_157','1' -'158','val_158','1' -'160','val_160','1' -'162','val_162','1' -'163','val_163','1' -'164','val_164','1' -'164','val_164','1' -'165','val_165','1' -'165','val_165','1' -'166','val_166','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'168','val_168','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'170','val_170','1' -'172','val_172','1' -'172','val_172','1' -'174','val_174','1' -'174','val_174','1' -'175','val_175','1' -'175','val_175','1' -'176','val_176','1' -'176','val_176','1' -'177','val_177','1' -'178','val_178','1' -'179','val_179','1' -'179','val_179','1' -'180','val_180','1' -'181','val_181','1' -'183','val_183','1' -'186','val_186','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'189','val_189','1' -'190','val_190','1' -'191','val_191','1' -'191','val_191','1' -'192','val_192','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'194','val_194','1' -'195','val_195','1' -'195','val_195','1' -'196','val_196','1' -'197','val_197','1' -'197','val_197','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'200','val_200','1' -'200','val_200','1' -'201','val_201','1' -'202','val_202','1' -'203','val_203','1' -'203','val_203','1' -'205','val_205','1' -'205','val_205','1' -'207','val_207','1' -'207','val_207','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'209','val_209','1' -'209','val_209','1' -'213','val_213','1' -'213','val_213','1' -'214','val_214','1' -'216','val_216','1' -'216','val_216','1' -'217','val_217','1' -'217','val_217','1' -'218','val_218','1' -'219','val_219','1' -'219','val_219','1' -'221','val_221','1' -'221','val_221','1' -'222','val_222','1' -'223','val_223','1' -'223','val_223','1' -'224','val_224','1' -'224','val_224','1' -'226','val_226','1' -'228','val_228','1' -'229','val_229','1' -'229','val_229','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'233','val_233','1' -'233','val_233','1' -'235','val_235','1' -'237','val_237','1' -'237','val_237','1' -'238','val_238','1' -'238','val_238','1' -'239','val_239','1' -'239','val_239','1' -'241','val_241','1' -'242','val_242','1' -'242','val_242','1' -'244','val_244','1' -'247','val_247','1' -'248','val_248','1' -'249','val_249','1' -'252','val_252','1' -'255','val_255','1' -'255','val_255','1' -'256','val_256','1' -'256','val_256','1' -'257','val_257','1' -'258','val_258','1' -'260','val_260','1' -'262','val_262','1' -'263','val_263','1' -'265','val_265','1' -'265','val_265','1' -'266','val_266','1' -'272','val_272','1' -'272','val_272','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'274','val_274','1' -'275','val_275','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'278','val_278','1' -'278','val_278','1' -'280','val_280','1' -'280','val_280','1' -'281','val_281','1' -'281','val_281','1' -'282','val_282','1' -'282','val_282','1' -'283','val_283','1' -'284','val_284','1' -'285','val_285','1' -'286','val_286','1' -'287','val_287','1' -'288','val_288','1' -'288','val_288','1' -'289','val_289','1' -'291','val_291','1' -'292','val_292','1' -'296','val_296','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'302','val_302','1' -'305','val_305','1' -'306','val_306','1' -'307','val_307','1' -'307','val_307','1' -'308','val_308','1' -'309','val_309','1' -'309','val_309','1' -'310','val_310','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'315','val_315','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'317','val_317','1' -'317','val_317','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'321','val_321','1' -'321','val_321','1' -'322','val_322','1' -'322','val_322','1' -'323','val_323','1' -'325','val_325','1' -'325','val_325','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'331','val_331','1' -'331','val_331','1' -'332','val_332','1' -'333','val_333','1' -'333','val_333','1' -'335','val_335','1' -'336','val_336','1' -'338','val_338','1' -'339','val_339','1' -'341','val_341','1' -'342','val_342','1' -'342','val_342','1' -'344','val_344','1' -'344','val_344','1' -'345','val_345','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'351','val_351','1' -'353','val_353','1' -'353','val_353','1' -'356','val_356','1' -'360','val_360','1' -'362','val_362','1' -'364','val_364','1' -'365','val_365','1' -'366','val_366','1' -'367','val_367','1' -'367','val_367','1' -'368','val_368','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'373','val_373','1' -'374','val_374','1' -'375','val_375','1' -'377','val_377','1' -'378','val_378','1' -'379','val_379','1' -'382','val_382','1' -'382','val_382','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'386','val_386','1' -'389','val_389','1' -'392','val_392','1' -'393','val_393','1' -'394','val_394','1' -'395','val_395','1' -'395','val_395','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'397','val_397','1' -'397','val_397','1' -'399','val_399','1' -'399','val_399','1' -'400','val_400','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'402','val_402','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'404','val_404','1' -'404','val_404','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'407','val_407','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'411','val_411','1' -'413','val_413','1' -'413','val_413','1' -'414','val_414','1' -'414','val_414','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'418','val_418','1' -'419','val_419','1' -'421','val_421','1' -'424','val_424','1' -'424','val_424','1' -'427','val_427','1' -'429','val_429','1' -'429','val_429','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'432','val_432','1' -'435','val_435','1' -'436','val_436','1' -'437','val_437','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'439','val_439','1' -'439','val_439','1' -'443','val_443','1' -'444','val_444','1' -'446','val_446','1' -'448','val_448','1' -'449','val_449','1' -'452','val_452','1' -'453','val_453','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'455','val_455','1' -'457','val_457','1' -'458','val_458','1' -'458','val_458','1' -'459','val_459','1' -'459','val_459','1' -'460','val_460','1' -'462','val_462','1' -'462','val_462','1' -'463','val_463','1' -'463','val_463','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'467','val_467','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'470','val_470','1' -'472','val_472','1' -'475','val_475','1' -'477','val_477','1' -'478','val_478','1' -'478','val_478','1' -'479','val_479','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'481','val_481','1' -'482','val_482','1' -'483','val_483','1' -'484','val_484','1' -'485','val_485','1' -'487','val_487','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'490','val_490','1' -'491','val_491','1' -'492','val_492','1' -'492','val_492','1' -'493','val_493','1' -'494','val_494','1' -'495','val_495','1' -'496','val_496','1' -'497','val_497','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -500 rows selected ->>> SELECT * FROM test_table3 ORDER BY key; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT * FROM test_table3 ORDER BY key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:test_table3.key, type:int, comment:null), FieldSchema(name:test_table3.value, type:string, comment:null), FieldSchema(name:test_table3.ds, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT * FROM test_table3 ORDER BY key -INFO : PREHOOK: query: SELECT * FROM test_table3 ORDER BY key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT * FROM test_table3 ORDER BY key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT * FROM test_table3 ORDER BY key -'test_table3.key','test_table3.value','test_table3.ds' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'0','val_0','1' -'2','val_2','1' -'4','val_4','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'5','val_5','1' -'8','val_8','1' -'9','val_9','1' -'10','val_10','1' -'11','val_11','1' -'12','val_12','1' -'12','val_12','1' -'12','val_12','1' -'12','val_12','1' -'15','val_15','1' -'15','val_15','1' -'15','val_15','1' -'15','val_15','1' -'17','val_17','1' -'18','val_18','1' -'18','val_18','1' -'18','val_18','1' -'18','val_18','1' -'19','val_19','1' -'20','val_20','1' -'24','val_24','1' -'24','val_24','1' -'24','val_24','1' -'24','val_24','1' -'26','val_26','1' -'26','val_26','1' -'26','val_26','1' -'26','val_26','1' -'27','val_27','1' -'28','val_28','1' -'30','val_30','1' -'33','val_33','1' -'34','val_34','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'35','val_35','1' -'37','val_37','1' -'37','val_37','1' -'37','val_37','1' -'37','val_37','1' -'41','val_41','1' -'42','val_42','1' -'42','val_42','1' -'42','val_42','1' -'42','val_42','1' -'43','val_43','1' -'44','val_44','1' -'47','val_47','1' -'51','val_51','1' -'51','val_51','1' -'51','val_51','1' -'51','val_51','1' -'53','val_53','1' -'54','val_54','1' -'57','val_57','1' -'58','val_58','1' -'58','val_58','1' -'58','val_58','1' -'58','val_58','1' -'64','val_64','1' -'65','val_65','1' -'66','val_66','1' -'67','val_67','1' -'67','val_67','1' -'67','val_67','1' -'67','val_67','1' -'69','val_69','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'70','val_70','1' -'72','val_72','1' -'72','val_72','1' -'72','val_72','1' -'72','val_72','1' -'74','val_74','1' -'76','val_76','1' -'76','val_76','1' -'76','val_76','1' -'76','val_76','1' -'77','val_77','1' -'78','val_78','1' -'80','val_80','1' -'82','val_82','1' -'83','val_83','1' -'83','val_83','1' -'83','val_83','1' -'83','val_83','1' -'84','val_84','1' -'84','val_84','1' -'84','val_84','1' -'84','val_84','1' -'85','val_85','1' -'86','val_86','1' -'87','val_87','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'90','val_90','1' -'92','val_92','1' -'95','val_95','1' -'95','val_95','1' -'95','val_95','1' -'95','val_95','1' -'96','val_96','1' -'97','val_97','1' -'97','val_97','1' -'97','val_97','1' -'97','val_97','1' -'98','val_98','1' -'98','val_98','1' -'98','val_98','1' -'98','val_98','1' -'100','val_100','1' -'100','val_100','1' -'100','val_100','1' -'100','val_100','1' -'103','val_103','1' -'103','val_103','1' -'103','val_103','1' -'103','val_103','1' -'104','val_104','1' -'104','val_104','1' -'104','val_104','1' -'104','val_104','1' -'105','val_105','1' -'111','val_111','1' -'113','val_113','1' -'113','val_113','1' -'113','val_113','1' -'113','val_113','1' -'114','val_114','1' -'116','val_116','1' -'118','val_118','1' -'118','val_118','1' -'118','val_118','1' -'118','val_118','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'119','val_119','1' -'120','val_120','1' -'120','val_120','1' -'120','val_120','1' -'120','val_120','1' -'125','val_125','1' -'125','val_125','1' -'125','val_125','1' -'125','val_125','1' -'126','val_126','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'128','val_128','1' -'129','val_129','1' -'129','val_129','1' -'129','val_129','1' -'129','val_129','1' -'131','val_131','1' -'133','val_133','1' -'134','val_134','1' -'134','val_134','1' -'134','val_134','1' -'134','val_134','1' -'136','val_136','1' -'137','val_137','1' -'137','val_137','1' -'137','val_137','1' -'137','val_137','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'138','val_138','1' -'143','val_143','1' -'145','val_145','1' -'146','val_146','1' -'146','val_146','1' -'146','val_146','1' -'146','val_146','1' -'149','val_149','1' -'149','val_149','1' -'149','val_149','1' -'149','val_149','1' -'150','val_150','1' -'152','val_152','1' -'152','val_152','1' -'152','val_152','1' -'152','val_152','1' -'153','val_153','1' -'155','val_155','1' -'156','val_156','1' -'157','val_157','1' -'158','val_158','1' -'160','val_160','1' -'162','val_162','1' -'163','val_163','1' -'164','val_164','1' -'164','val_164','1' -'164','val_164','1' -'164','val_164','1' -'165','val_165','1' -'165','val_165','1' -'165','val_165','1' -'165','val_165','1' -'166','val_166','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'167','val_167','1' -'168','val_168','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'169','val_169','1' -'170','val_170','1' -'172','val_172','1' -'172','val_172','1' -'172','val_172','1' -'172','val_172','1' -'174','val_174','1' -'174','val_174','1' -'174','val_174','1' -'174','val_174','1' -'175','val_175','1' -'175','val_175','1' -'175','val_175','1' -'175','val_175','1' -'176','val_176','1' -'176','val_176','1' -'176','val_176','1' -'176','val_176','1' -'177','val_177','1' -'178','val_178','1' -'179','val_179','1' -'179','val_179','1' -'179','val_179','1' -'179','val_179','1' -'180','val_180','1' -'181','val_181','1' -'183','val_183','1' -'186','val_186','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'187','val_187','1' -'189','val_189','1' -'190','val_190','1' -'191','val_191','1' -'191','val_191','1' -'191','val_191','1' -'191','val_191','1' -'192','val_192','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'194','val_194','1' -'195','val_195','1' -'195','val_195','1' -'195','val_195','1' -'195','val_195','1' -'196','val_196','1' -'197','val_197','1' -'197','val_197','1' -'197','val_197','1' -'197','val_197','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'199','val_199','1' -'200','val_200','1' -'200','val_200','1' -'200','val_200','1' -'200','val_200','1' -'201','val_201','1' -'202','val_202','1' -'203','val_203','1' -'203','val_203','1' -'203','val_203','1' -'203','val_203','1' -'205','val_205','1' -'205','val_205','1' -'205','val_205','1' -'205','val_205','1' -'207','val_207','1' -'207','val_207','1' -'207','val_207','1' -'207','val_207','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'208','val_208','1' -'209','val_209','1' -'209','val_209','1' -'209','val_209','1' -'209','val_209','1' -'213','val_213','1' -'213','val_213','1' -'213','val_213','1' -'213','val_213','1' -'214','val_214','1' -'216','val_216','1' -'216','val_216','1' -'216','val_216','1' -'216','val_216','1' -'217','val_217','1' -'217','val_217','1' -'217','val_217','1' -'217','val_217','1' -'218','val_218','1' -'219','val_219','1' -'219','val_219','1' -'219','val_219','1' -'219','val_219','1' -'221','val_221','1' -'221','val_221','1' -'221','val_221','1' -'221','val_221','1' -'222','val_222','1' -'223','val_223','1' -'223','val_223','1' -'223','val_223','1' -'223','val_223','1' -'224','val_224','1' -'224','val_224','1' -'224','val_224','1' -'224','val_224','1' -'226','val_226','1' -'228','val_228','1' -'229','val_229','1' -'229','val_229','1' -'229','val_229','1' -'229','val_229','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'230','val_230','1' -'233','val_233','1' -'233','val_233','1' -'233','val_233','1' -'233','val_233','1' -'235','val_235','1' -'237','val_237','1' -'237','val_237','1' -'237','val_237','1' -'237','val_237','1' -'238','val_238','1' -'238','val_238','1' -'238','val_238','1' -'238','val_238','1' -'239','val_239','1' -'239','val_239','1' -'239','val_239','1' -'239','val_239','1' -'241','val_241','1' -'242','val_242','1' -'242','val_242','1' -'242','val_242','1' -'242','val_242','1' -'244','val_244','1' -'247','val_247','1' -'248','val_248','1' -'249','val_249','1' -'252','val_252','1' -'255','val_255','1' -'255','val_255','1' -'255','val_255','1' -'255','val_255','1' -'256','val_256','1' -'256','val_256','1' -'256','val_256','1' -'256','val_256','1' -'257','val_257','1' -'258','val_258','1' -'260','val_260','1' -'262','val_262','1' -'263','val_263','1' -'265','val_265','1' -'265','val_265','1' -'265','val_265','1' -'265','val_265','1' -'266','val_266','1' -'272','val_272','1' -'272','val_272','1' -'272','val_272','1' -'272','val_272','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'274','val_274','1' -'275','val_275','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'277','val_277','1' -'278','val_278','1' -'278','val_278','1' -'278','val_278','1' -'278','val_278','1' -'280','val_280','1' -'280','val_280','1' -'280','val_280','1' -'280','val_280','1' -'281','val_281','1' -'281','val_281','1' -'281','val_281','1' -'281','val_281','1' -'282','val_282','1' -'282','val_282','1' -'282','val_282','1' -'282','val_282','1' -'283','val_283','1' -'284','val_284','1' -'285','val_285','1' -'286','val_286','1' -'287','val_287','1' -'288','val_288','1' -'288','val_288','1' -'288','val_288','1' -'288','val_288','1' -'289','val_289','1' -'291','val_291','1' -'292','val_292','1' -'296','val_296','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'298','val_298','1' -'302','val_302','1' -'305','val_305','1' -'306','val_306','1' -'307','val_307','1' -'307','val_307','1' -'307','val_307','1' -'307','val_307','1' -'308','val_308','1' -'309','val_309','1' -'309','val_309','1' -'309','val_309','1' -'309','val_309','1' -'310','val_310','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'311','val_311','1' -'315','val_315','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'316','val_316','1' -'317','val_317','1' -'317','val_317','1' -'317','val_317','1' -'317','val_317','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'318','val_318','1' -'321','val_321','1' -'321','val_321','1' -'321','val_321','1' -'321','val_321','1' -'322','val_322','1' -'322','val_322','1' -'322','val_322','1' -'322','val_322','1' -'323','val_323','1' -'325','val_325','1' -'325','val_325','1' -'325','val_325','1' -'325','val_325','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'327','val_327','1' -'331','val_331','1' -'331','val_331','1' -'331','val_331','1' -'331','val_331','1' -'332','val_332','1' -'333','val_333','1' -'333','val_333','1' -'333','val_333','1' -'333','val_333','1' -'335','val_335','1' -'336','val_336','1' -'338','val_338','1' -'339','val_339','1' -'341','val_341','1' -'342','val_342','1' -'342','val_342','1' -'342','val_342','1' -'342','val_342','1' -'344','val_344','1' -'344','val_344','1' -'344','val_344','1' -'344','val_344','1' -'345','val_345','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'348','val_348','1' -'351','val_351','1' -'353','val_353','1' -'353','val_353','1' -'353','val_353','1' -'353','val_353','1' -'356','val_356','1' -'360','val_360','1' -'362','val_362','1' -'364','val_364','1' -'365','val_365','1' -'366','val_366','1' -'367','val_367','1' -'367','val_367','1' -'367','val_367','1' -'367','val_367','1' -'368','val_368','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'373','val_373','1' -'374','val_374','1' -'375','val_375','1' -'377','val_377','1' -'378','val_378','1' -'379','val_379','1' -'382','val_382','1' -'382','val_382','1' -'382','val_382','1' -'382','val_382','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'384','val_384','1' -'386','val_386','1' -'389','val_389','1' -'392','val_392','1' -'393','val_393','1' -'394','val_394','1' -'395','val_395','1' -'395','val_395','1' -'395','val_395','1' -'395','val_395','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'396','val_396','1' -'397','val_397','1' -'397','val_397','1' -'397','val_397','1' -'397','val_397','1' -'399','val_399','1' -'399','val_399','1' -'399','val_399','1' -'399','val_399','1' -'400','val_400','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'402','val_402','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'403','val_403','1' -'404','val_404','1' -'404','val_404','1' -'404','val_404','1' -'404','val_404','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'406','val_406','1' -'407','val_407','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'409','val_409','1' -'411','val_411','1' -'413','val_413','1' -'413','val_413','1' -'413','val_413','1' -'413','val_413','1' -'414','val_414','1' -'414','val_414','1' -'414','val_414','1' -'414','val_414','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'418','val_418','1' -'419','val_419','1' -'421','val_421','1' -'424','val_424','1' -'424','val_424','1' -'424','val_424','1' -'424','val_424','1' -'427','val_427','1' -'429','val_429','1' -'429','val_429','1' -'429','val_429','1' -'429','val_429','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'430','val_430','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'431','val_431','1' -'432','val_432','1' -'435','val_435','1' -'436','val_436','1' -'437','val_437','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'438','val_438','1' -'439','val_439','1' -'439','val_439','1' -'439','val_439','1' -'439','val_439','1' -'443','val_443','1' -'444','val_444','1' -'446','val_446','1' -'448','val_448','1' -'449','val_449','1' -'452','val_452','1' -'453','val_453','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'454','val_454','1' -'455','val_455','1' -'457','val_457','1' -'458','val_458','1' -'458','val_458','1' -'458','val_458','1' -'458','val_458','1' -'459','val_459','1' -'459','val_459','1' -'459','val_459','1' -'459','val_459','1' -'460','val_460','1' -'462','val_462','1' -'462','val_462','1' -'462','val_462','1' -'462','val_462','1' -'463','val_463','1' -'463','val_463','1' -'463','val_463','1' -'463','val_463','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'466','val_466','1' -'467','val_467','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'468','val_468','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'469','val_469','1' -'470','val_470','1' -'472','val_472','1' -'475','val_475','1' -'477','val_477','1' -'478','val_478','1' -'478','val_478','1' -'478','val_478','1' -'478','val_478','1' -'479','val_479','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'480','val_480','1' -'481','val_481','1' -'482','val_482','1' -'483','val_483','1' -'484','val_484','1' -'485','val_485','1' -'487','val_487','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'489','val_489','1' -'490','val_490','1' -'491','val_491','1' -'492','val_492','1' -'492','val_492','1' -'492','val_492','1' -'492','val_492','1' -'493','val_493','1' -'494','val_494','1' -'495','val_495','1' -'496','val_496','1' -'497','val_497','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -'498','val_498','1' -1,028 rows selected ->>> EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16); -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-2:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: test_table1' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: true' -' predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean)' -' sampleDesc: BUCKET 2 OUT OF 16' -' Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE' -' Select Operator' -' expressions: key (type: int), value (type: string), ds (type: string)' -' outputColumnNames: _col0, _col1, _col2' -' Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE' -' File Output Operator' -' compressed: false' -' GlobalTableId: 0' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' properties:' -' columns _col0,_col1,_col2' -' columns.types int:string:string' -' escape.delim \' -' hive.serialization.extend.additional.nesting.levels true' -' serialization.escape.crlf true' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' TotalFiles: 1' -' GatherStats: false' -' MultiFileSpray: false' -' Path -> Alias:' -' file:/!!ELIDED!! [test_table1]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: 000001_0' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' partition values:' -' ds 1' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table1' -' numFiles 16' -' numRows 500' -' partition_columns ds' -' partition_columns.types string' -' rawDataSize 5312' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table1' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_11.test_table1' -' name: smb_mapjoin_11.test_table1' -' Truncated Path -> Alias:' -' /smb_mapjoin_11.db/test_table1/ds=1/000001_0 [test_table1]' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -109 rows selected ->>> EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16); -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-2:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: test_table3' -' Statistics: Num rows: 1028 Data size: 10968 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: true' -' predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean)' -' sampleDesc: BUCKET 2 OUT OF 16' -' Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE' -' Select Operator' -' expressions: key (type: int), value (type: string), ds (type: string)' -' outputColumnNames: _col0, _col1, _col2' -' Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE' -' File Output Operator' -' compressed: false' -' GlobalTableId: 0' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' properties:' -' columns _col0,_col1,_col2' -' columns.types int:string:string' -' escape.delim \' -' hive.serialization.extend.additional.nesting.levels true' -' serialization.escape.crlf true' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' TotalFiles: 1' -' GatherStats: false' -' MultiFileSpray: false' -' Path -> Alias:' -' file:/!!ELIDED!! [test_table3]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: 000001_0' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' partition values:' -' ds 1' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table3' -' numFiles 16' -' numRows 1028' -' partition_columns ds' -' partition_columns.types string' -' rawDataSize 10968' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 11996' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_11.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_11.test_table3' -' name: smb_mapjoin_11.test_table3' -' Truncated Path -> Alias:' -' /smb_mapjoin_11.db/test_table3/ds=1/000001_0 [test_table3]' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -108 rows selected ->>> SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16); -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:test_table1.key, type:int, comment:null), FieldSchema(name:test_table1.value, type:string, comment:null), FieldSchema(name:test_table1.ds, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) -'test_table1.key','test_table1.value','test_table1.ds' -'17','val_17','1' -'33','val_33','1' -'65','val_65','1' -'97','val_97','1' -'97','val_97','1' -'113','val_113','1' -'113','val_113','1' -'129','val_129','1' -'129','val_129','1' -'145','val_145','1' -'177','val_177','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'209','val_209','1' -'209','val_209','1' -'241','val_241','1' -'257','val_257','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'289','val_289','1' -'305','val_305','1' -'321','val_321','1' -'321','val_321','1' -'353','val_353','1' -'353','val_353','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'449','val_449','1' -'481','val_481','1' -'497','val_497','1' -41 rows selected ->>> SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16); -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:test_table3.key, type:int, comment:null), FieldSchema(name:test_table3.value, type:string, comment:null), FieldSchema(name:test_table3.ds, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) -'test_table3.key','test_table3.value','test_table3.ds' -'497','val_497','1' -'481','val_481','1' -'449','val_449','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'417','val_417','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'401','val_401','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'369','val_369','1' -'353','val_353','1' -'353','val_353','1' -'353','val_353','1' -'353','val_353','1' -'321','val_321','1' -'321','val_321','1' -'321','val_321','1' -'321','val_321','1' -'305','val_305','1' -'289','val_289','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'273','val_273','1' -'257','val_257','1' -'241','val_241','1' -'209','val_209','1' -'209','val_209','1' -'209','val_209','1' -'209','val_209','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'193','val_193','1' -'177','val_177','1' -'145','val_145','1' -'129','val_129','1' -'129','val_129','1' -'129','val_129','1' -'129','val_129','1' -'113','val_113','1' -'113','val_113','1' -'113','val_113','1' -'113','val_113','1' -'97','val_97','1' -'97','val_97','1' -'97','val_97','1' -'97','val_97','1' -'65','val_65','1' -'33','val_33','1' -'17','val_17','1' -97 rows selected ->>> ->>> -- Join data from a sampled bucket to verify the data is bucketed ->>> SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_c0, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 2 -INFO : Launching Job 1 out of 2 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks not specified. Estimated from input data size: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 2 out of 2 -INFO : Starting task [Stage-2:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table1@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_11@test_table3@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-2: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -'_c0' -'293' -1 row selected ->>> ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col7 + Position of Big Table: 0 + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: int) + tag: -1 + value expressions: _col0 (type: int), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Truncated Path -> Alias: + /test_table1/ds=1 [a] + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 16 + Static Partition Specification: ds=1/ +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 16 + GatherStats: true + MultiFileSpray: true + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +PREHOOK: Input: default@test_table2 +PREHOOK: Input: default@test_table2@ds=1 +PREHOOK: Output: default@test_table3@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +POSTHOOK: Input: default@test_table2 +POSTHOOK: Input: default@test_table2@ds=1 +POSTHOOK: Output: default@test_table3@ds=1 +POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM test_table1 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: SELECT * FROM test_table3 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_table1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: 000001_0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Truncated Path -> Alias: + /test_table1/ds=1/000001_0 [test_table1] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_table3 + Statistics: Num rows: 1028 Data size: 10968 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: 000001_0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 1028 + partition_columns ds + partition_columns.types string + rawDataSize 10968 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11996 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + name: default.test_table3 + Truncated Path -> Alias: + /test_table3/ds=1/000001_0 [test_table3] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +17 val_17 1 +33 val_33 1 +65 val_65 1 +97 val_97 1 +97 val_97 1 +113 val_113 1 +113 val_113 1 +129 val_129 1 +129 val_129 1 +145 val_145 1 +177 val_177 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +209 val_209 1 +209 val_209 1 +241 val_241 1 +257 val_257 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +289 val_289 1 +305 val_305 1 +321 val_321 1 +321 val_321 1 +353 val_353 1 +353 val_353 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +449 val_449 1 +481 val_481 1 +497 val_497 1 +PREHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +497 val_497 1 +481 val_481 1 +449 val_449 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +305 val_305 1 +289 val_289 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +257 val_257 1 +241 val_241 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +177 val_177 1 +145 val_145 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +65 val_65 1 +33 val_33 1 +17 val_17 1 +PREHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +293 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out index 98bf25e..9928a60 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out @@ -1,822 +1,430 @@ ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> ->>> set hive.exec.reducers.max = 1; -No rows affected ->>> set hive.merge.mapfiles=false; -No rows affected ->>> set hive.merge.mapredfiles=false; -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> -- This test verifies that the output of a sort merge join on 1 big partition with multiple small partitions is bucketed and sorted ->>> ->>> -- Create two bucketed and sorted tables ->>> CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_12 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_12 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -No rows affected ->>> CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_12 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_12 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -No rows affected ->>> ->>> FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT *; -INFO : Compiling commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT * -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_col0, type:int, comment:null), FieldSchema(name:_col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT * -INFO : PREHOOK: query: FROM default.src -INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * -INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT * -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: default@src -INFO : PREHOOK: Output: smb_mapjoin_12@test_table1@ds=1 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table2@ds=1 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table2@ds=2 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table2@ds=3 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 4 -INFO : Launching Job 1 out of 4 -INFO : Starting task [Stage-4:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table1 partition (ds=1) from file:/!!ELIDED!! -INFO : Launching Job 2 out of 4 -INFO : Starting task [Stage-6:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 3 out of 4 -INFO : Starting task [Stage-8:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 4 out of 4 -INFO : Starting task [Stage-10:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-5:STATS] in serial mode -INFO : Starting task [Stage-1:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table2 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-2:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table2 partition (ds=2) from file:/!!ELIDED!! -INFO : Starting task [Stage-3:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table2 partition (ds=3) from file:/!!ELIDED!! -INFO : Starting task [Stage-7:STATS] in serial mode -INFO : Starting task [Stage-9:STATS] in serial mode -INFO : Starting task [Stage-11:STATS] in serial mode -INFO : POSTHOOK: query: FROM default.src +PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table1 +POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table1 +PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table2 +POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table2 +PREHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT * -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: default@src -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table1@ds=1 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table2@ds=1 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table2@ds=2 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table2@ds=3 -INFO : POSTHOOK: Lineage: test_table1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2 PARTITION(ds=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-4: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-6: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-8: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-10: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query FROM default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_table1@ds=1 +PREHOOK: Output: default@test_table2@ds=1 +PREHOOK: Output: default@test_table2@ds=2 +PREHOOK: Output: default@test_table2@ds=3 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT * INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '3') SELECT * -No rows affected ->>> ->>> ->>> ->>> ->>> -- Create a bucketed table ->>> CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_12 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_12 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS -No rows affected ->>> ->>> -- Insert data into the bucketed table by joining the two bucketed and sorted tables, bucketing is not enforced ->>> EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : PREHOOK: query: EXPLAIN EXTENDED -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-4:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_table1@ds=1 +POSTHOOK: Output: default@test_table2@ds=1 +POSTHOOK: Output: default@test_table2@ds=2 +POSTHOOK: Output: default@test_table2@ds=3 +POSTHOOK: Lineage: test_table1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table3 +POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) PARTITIONED BY (ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table3 +PREHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -' Stage-2 depends on stages: Stage-0' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col7' -' Position of Big Table: 0' -' BucketMapJoin: true' -' Select Operator' -' expressions: _col0 (type: int), _col7 (type: string)' -' outputColumnNames: _col0, _col1' -' File Output Operator' -' compressed: false' -' GlobalTableId: 1' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Static Partition Specification: ds=1/' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table3' -' TotalFiles: 1' -' GatherStats: true' -' MultiFileSpray: false' -' Path -> Alias:' -' file:/!!ELIDED!! [a]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: ds=1' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' partition values:' -' ds 1' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table1' -' numFiles 16' -' numRows 500' -' partition_columns ds' -' partition_columns.types string' -' rawDataSize 5312' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table1' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table1' -' name: smb_mapjoin_12.test_table1' -' Truncated Path -> Alias:' -' /smb_mapjoin_12.db/test_table1/ds=1 [a]' -'' -' Stage: Stage-0' -' Move Operator' -' tables:' -' partition:' -' ds 1' -' replace: true' -' source: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table3' -'' -' Stage: Stage-2' -' Stats-Aggr Operator' -' Stats Aggregation Key Prefix: file:/!!ELIDED!! -'' -157 rows selected ->>> ->>> INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table2 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table2@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table2@ds=2 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table2@ds=3 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table3@ds=1 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table3 partition (ds=1) from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table2 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table2@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table2@ds=2 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table2@ds=3 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table3@ds=1 -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' -No rows affected ->>> ->>> -- Join data from a sampled bucket to verify the data is bucketed ->>> SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_c0, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3@ds=1 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 2 -INFO : Launching Job 1 out of 2 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks not specified. Estimated from input data size: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 2 out of 2 -INFO : Starting task [Stage-2:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3@ds=1 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-2: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -'_c0' -'879' -1 row selected ->>> ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> -- Join data from the sampled buckets of 2 tables to verify the data is bucketed and sorted ->>> explain extended -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain extended -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain extended +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col7 + Position of Big Table: 0 + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=1/ +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Truncated Path -> Alias: + /test_table1/ds=1 [a] + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +PREHOOK: Input: default@test_table2 +PREHOOK: Input: default@test_table2@ds=1 +PREHOOK: Input: default@test_table2@ds=2 +PREHOOK: Input: default@test_table2@ds=3 +PREHOOK: Output: default@test_table3@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +POSTHOOK: Input: default@test_table2 +POSTHOOK: Input: default@test_table2@ds=1 +POSTHOOK: Input: default@test_table2@ds=2 +POSTHOOK: Input: default@test_table2@ds=3 +POSTHOOK: Output: default@test_table3@ds=1 +POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +879 +PREHOOK: query: explain extended INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: query: explain extended +PREHOOK: type: QUERY +POSTHOOK: query: explain extended INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-4:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain extended -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain extended -INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -' Stage-2 depends on stages: Stage-0' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 3084 Data size: 32904 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 3084 Data size: 32904 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col7' -' Position of Big Table: 0' -' BucketMapJoin: true' -' Select Operator' -' expressions: _col0 (type: int), concat(_col1, _col7) (type: string)' -' outputColumnNames: _col0, _col1' -' File Output Operator' -' compressed: false' -' GlobalTableId: 1' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Static Partition Specification: ds=2/' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table3' -' TotalFiles: 1' -' GatherStats: true' -' MultiFileSpray: false' -' Path -> Alias:' -' file:/!!ELIDED!! [a]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: ds=1' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' partition values:' -' ds 1' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' numFiles 16' -' numRows 3084' -' partition_columns ds' -' partition_columns.types string' -' rawDataSize 32904' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 35988' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table3' -' name: smb_mapjoin_12.test_table3' -' Truncated Path -> Alias:' -' /smb_mapjoin_12.db/test_table3/ds=1 [a]' -'' -' Stage: Stage-0' -' Move Operator' -' tables:' -' partition:' -' ds 2' -' replace: true' -' source: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_12.test_table3' -' partition_columns ds' -' partition_columns.types string' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_12.test_table3' -'' -' Stage: Stage-2' -' Stats-Aggr Operator' -' Stats Aggregation Key Prefix: file:/!!ELIDED!! -'' -157 rows selected ->>> ->>> INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1'; -INFO : Compiling commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:_c2, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') -SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3@ds=1 -INFO : PREHOOK: Output: smb_mapjoin_12@test_table3@ds=2 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_12.test_table3 partition (ds=2) from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 3084 Data size: 32904 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3084 Data size: 32904 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), concat(_col1, _col7) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 3084 + partition_columns ds + partition_columns.types string + rawDataSize 32904 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 35988 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + name: default.test_table3 + Truncated Path -> Alias: + /test_table3/ds=1 [a] + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table1@ds=1 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3@ds=1 -INFO : POSTHOOK: Output: smb_mapjoin_12@test_table3@ds=2 -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=2).key SIMPLE [(test_table3)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: test_table3 PARTITION(ds=2).value EXPRESSION [(test_table3)a.FieldSchema(name:value, type:string, comment:null), (test_table1)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +PREHOOK: Output: default@test_table3@ds=2 +POSTHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') SELECT /*+mapjoin(b)*/ a.key, concat(a.value, b.value) FROM test_table3 a JOIN test_table1 b ON a.key = b.key AND a.ds = '1' AND b.ds='1' -No rows affected ->>> ->>> SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2'; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_c0, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' -INFO : PREHOOK: query: SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_12@test_table3@ds=2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_12@test_table3@ds=2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' -'_c0' -'879' -1 row selected ->>> !record +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +POSTHOOK: Output: default@test_table3@ds=2 +POSTHOOK: Lineage: test_table3 PARTITION(ds=2).key SIMPLE [(test_table3)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: test_table3 PARTITION(ds=2).value EXPRESSION [(test_table3)a.FieldSchema(name:value, type:string, comment:null), (test_table1)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) from test_table3 tablesample (bucket 2 out of 16) a where ds = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=2 +#### A masked pattern was here #### +879 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out index d303900..49ff635 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out @@ -1,687 +1,388 @@ ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> ->>> set hive.exec.reducers.max = 1; -No rows affected ->>> set hive.merge.mapfiles=false; -No rows affected ->>> set hive.merge.mapredfiles=false; -No rows affected ->>> ->>> -- This test verifies that the sort merge join optimizer works when the tables are joined on columns with different names ->>> ->>> -- Create bucketed and sorted tables ->>> CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_13 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_13 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS -No rows affected ->>> CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_13 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_13 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS -No rows affected ->>> CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_13 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_13 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS -No rows affected ->>> CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_13 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table4 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_13 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table4 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS -No rows affected ->>> ->>> FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INSERT OVERWRITE TABLE test_table3 SELECT * -INSERT OVERWRITE TABLE test_table4 SELECT *; -INFO : Compiling commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INSERT OVERWRITE TABLE test_table3 SELECT * -INSERT OVERWRITE TABLE test_table4 SELECT * -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_col0, type:int, comment:null), FieldSchema(name:_col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INSERT OVERWRITE TABLE test_table3 SELECT * -INSERT OVERWRITE TABLE test_table4 SELECT * -INFO : PREHOOK: query: FROM default.src +PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table1 +POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key ASC) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table1 +PREHOOK: query: CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table2 +POSTHOOK: query: CREATE TABLE test_table2 (value INT, key STRING) CLUSTERED BY (value) SORTED BY (value ASC) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table2 +PREHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table3 +POSTHOOK: query: CREATE TABLE test_table3 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (key ASC, value ASC) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table3 +PREHOOK: query: CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table4 +POSTHOOK: query: CREATE TABLE test_table4 (key INT, value STRING) CLUSTERED BY (key, value) SORTED BY (value ASC, key ASC) INTO 16 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table4 +PREHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 SELECT * INSERT OVERWRITE TABLE test_table2 SELECT * INSERT OVERWRITE TABLE test_table3 SELECT * INSERT OVERWRITE TABLE test_table4 SELECT * -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: default@src -INFO : PREHOOK: Output: smb_mapjoin_13@test_table1 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table2 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table3 -INFO : PREHOOK: Output: smb_mapjoin_13@test_table4 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 4 -INFO : Launching Job 1 out of 4 -INFO : Starting task [Stage-4:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_13.test_table1 from file:/!!ELIDED!! -INFO : Launching Job 2 out of 4 -INFO : Starting task [Stage-6:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 3 out of 4 -INFO : Starting task [Stage-8:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Launching Job 4 out of 4 -INFO : Starting task [Stage-10:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-5:STATS] in serial mode -INFO : Starting task [Stage-1:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_13.test_table2 from file:/!!ELIDED!! -INFO : Starting task [Stage-2:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_13.test_table3 from file:/!!ELIDED!! -INFO : Starting task [Stage-3:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_13.test_table4 from file:/!!ELIDED!! -INFO : Starting task [Stage-7:STATS] in serial mode -INFO : Starting task [Stage-9:STATS] in serial mode -INFO : Starting task [Stage-11:STATS] in serial mode -INFO : POSTHOOK: query: FROM default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_table1 +PREHOOK: Output: default@test_table2 +PREHOOK: Output: default@test_table3 +PREHOOK: Output: default@test_table4 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 SELECT * INSERT OVERWRITE TABLE test_table2 SELECT * INSERT OVERWRITE TABLE test_table3 SELECT * INSERT OVERWRITE TABLE test_table4 SELECT * -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: default@src -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table1 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table2 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table3 -INFO : POSTHOOK: Output: smb_mapjoin_13@test_table4 -INFO : POSTHOOK: Lineage: test_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2.key SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2.value EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-4: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-6: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-8: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-10: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INSERT OVERWRITE TABLE test_table3 SELECT * -INSERT OVERWRITE TABLE test_table4 SELECT * -No rows affected ->>> ->>> -- Join data from 2 tables on their respective sorted columns (one each, with different names) and ->>> -- verify sort merge join is used ->>> EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10; -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_table1 +POSTHOOK: Output: default@test_table2 +POSTHOOK: Output: default@test_table3 +POSTHOOK: Output: default@test_table4 +POSTHOOK: Lineage: test_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2.key SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2.value EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN EXTENDED SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 value (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Position of Big Table: 0' -' BucketMapJoin: true' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' Reduce Output Operator' -' key expressions: _col0 (type: int)' -' null sort order: a' -' sort order: +' -' tag: -1' -' TopN: 10' -' TopN Hash Memory Usage: 0.1' -' value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string)' -' auto parallelism: false' -' Path -> Alias:' -' file:/!!ELIDED!! [a]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: test_table1' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_13.test_table1' -' numFiles 16' -' numRows 500' -' rawDataSize 5312' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_13.test_table1' -' numFiles 16' -' numRows 500' -' rawDataSize 5312' -' serialization.ddl struct test_table1 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_13.test_table1' -' name: smb_mapjoin_13.test_table1' -' Truncated Path -> Alias:' -' /smb_mapjoin_13.db/test_table1 [a]' -' Needs Tagging: false' -' Reduce Operator Tree:' -' Select Operator' -' expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' Limit' -' Number of rows: 10' -' File Output Operator' -' compressed: false' -' GlobalTableId: 0' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' properties:' -' columns _col0,_col1,_col2,_col3' -' columns.types int:string:int:string' -' escape.delim \' -' hive.serialization.extend.additional.nesting.levels true' -' serialization.escape.crlf true' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' TotalFiles: 1' -' GatherStats: false' -' MultiFileSpray: false' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: 10' -' Processor Tree:' -' ListSink' -'' -131 rows selected ->>> ->>> SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.value, type:int, comment:null), FieldSchema(name:b.key, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_13@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_13@test_table2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_13@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_13@test_table2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 -'a.key','a.value','b.value','b.key' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'0','val_0','0','val_0' -'2','val_2','2','val_2' -10 rows selected ->>> ->>> -- Join data from 2 tables on their respective columns (two each, with the same names but sorted ->>> -- with different priorities) and verify sort merge join is not used ->>> EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10; -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: query: EXPLAIN EXTENDED -SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-4:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 value (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: test_table1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Truncated Path -> Alias: + /test_table1 [a] + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Limit + Number of rows: 10 + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table1 a JOIN test_table2 b ON a.key = b.value ORDER BY a.key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table2 +#### A masked pattern was here #### +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +2 val_2 2 val_2 +PREHOOK: query: EXPLAIN EXTENDED SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN EXTENDED +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -'Explain' -'STAGE DEPENDENCIES:' -' Stage-3 is a root stage' -' Stage-1 depends on stages: Stage-3' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-3' -' Map Reduce Local Work' -' Alias -> Map Local Tables:' -' b ' -' Fetch Operator' -' limit: -1' -' Alias -> Map Local Operator Tree:' -' b ' -' TableScan' -' alias: b' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: UDFToDouble(value) is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' HashTable Sink Operator' -' keys:' -' 0 UDFToDouble(key) (type: double)' -' 1 UDFToDouble(value) (type: double)' -' Position of Big Table: 0' -'' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' GatherStats: false' -' Filter Operator' -' isSamplingPred: false' -' predicate: UDFToDouble(key) is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 UDFToDouble(key) (type: double)' -' 1 UDFToDouble(value) (type: double)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Position of Big Table: 0' -' Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE' -' Reduce Output Operator' -' key expressions: _col0 (type: int)' -' null sort order: a' -' sort order: +' -' Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE' -' tag: -1' -' TopN: 10' -' TopN Hash Memory Usage: 0.1' -' value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string)' -' auto parallelism: false' -' Local Work:' -' Map Reduce Local Work' -' Path -> Alias:' -' file:/!!ELIDED!! [a]' -' Path -> Partition:' -' file:/!!ELIDED!! ' -' Partition' -' base file name: test_table3' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_13.test_table3' -' numFiles 16' -' numRows 500' -' rawDataSize 5312' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' ' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' properties:' -' COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}' -' SORTBUCKETCOLSPREFIX TRUE' -' bucket_count 16' -' bucket_field_name key' -' column.name.delimiter ,' -' columns key,value' -' columns.comments ' -' columns.types int:string' -' file.inputformat org.apache.hadoop.mapred.TextInputFormat' -' file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' location file:/!!ELIDED!! -' name smb_mapjoin_13.test_table3' -' numFiles 16' -' numRows 500' -' rawDataSize 5312' -' serialization.ddl struct test_table3 { i32 key, string value}' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' totalSize 5812' -' transient_lastDdlTime !!UNIXTIME!!' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_13.test_table3' -' name: smb_mapjoin_13.test_table3' -' Truncated Path -> Alias:' -' /smb_mapjoin_13.db/test_table3 [a]' -' Needs Tagging: false' -' Reduce Operator Tree:' -' Select Operator' -' expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE' -' Limit' -' Number of rows: 10' -' Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE' -' File Output Operator' -' compressed: false' -' GlobalTableId: 0' -' directory: file:/!!ELIDED!! -' NumFilesPerFileSink: 1' -' Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE' -' Stats Publishing Key Prefix: file:/!!ELIDED!! -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' properties:' -' columns _col0,_col1,_col2,_col3' -' columns.types int:string:int:string' -' escape.delim \' -' hive.serialization.extend.additional.nesting.levels true' -' serialization.escape.crlf true' -' serialization.format 1' -' serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' TotalFiles: 1' -' GatherStats: false' -' MultiFileSpray: false' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: 10' -' Processor Tree:' -' ListSink' -'' -161 rows selected ->>> ->>> SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_13@test_table3 -INFO : PREHOOK: Input: smb_mapjoin_13@test_table4 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Starting task [Stage-3:MAPREDLOCAL] in serial mode -INFO : Starting to launch local task to process map join; maximum memory = !!ELIDED!! -INFO : End of local task; Time taken: !!ELIDED!! sec. -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:16 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_13@test_table3 -INFO : POSTHOOK: Input: smb_mapjoin_13@test_table4 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 -'a.key','a.value','b.key','b.value' -No rows selected ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(value) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(value) (type: double) + Position of Big Table: 0 + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(value) (type: double) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + auto parallelism: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: test_table3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 500 + rawDataSize 5312 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + column.name.delimiter , + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 500 + rawDataSize 5312 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + name: default.test_table3 + Truncated Path -> Alias: + /test_table3 [a] + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM test_table3 a JOIN test_table4 b ON a.key = b.value ORDER BY a.key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table4 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_16.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_16.q.out index b8a06dc..ab2b323 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_16.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_16.q.out @@ -1,254 +1,96 @@ ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> ->>> set hive.exec.reducers.max = 1; -No rows affected ->>> set hive.merge.mapfiles=false; -No rows affected ->>> set hive.merge.mapredfiles=false; -No rows affected ->>> ->>> -- Create bucketed and sorted tables ->>> CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_16 -INFO : PREHOOK: Output: smb_mapjoin_16@test_table1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_16 -INFO : POSTHOOK: Output: smb_mapjoin_16@test_table1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -No rows affected ->>> CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_16 -INFO : PREHOOK: Output: smb_mapjoin_16@test_table2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_16 -INFO : POSTHOOK: Output: smb_mapjoin_16@test_table2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -No rows affected ->>> ->>> FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT *; -INFO : Compiling commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_col0, type:int, comment:null), FieldSchema(name:_col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -INFO : PREHOOK: query: FROM default.src +PREHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table1 +POSTHOOK: query: CREATE TABLE test_table1 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table1 +PREHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table2 +POSTHOOK: query: CREATE TABLE test_table2 (key INT, value STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table2 +PREHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 SELECT * INSERT OVERWRITE TABLE test_table2 SELECT * -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: default@src -INFO : PREHOOK: Output: smb_mapjoin_16@test_table1 -INFO : PREHOOK: Output: smb_mapjoin_16@test_table2 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 2 -INFO : Launching Job 1 out of 2 -INFO : Starting task [Stage-2:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_16.test_table1 from file:/!!ELIDED!! -INFO : Launching Job 2 out of 2 -INFO : Starting task [Stage-4:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-3:STATS] in serial mode -INFO : Starting task [Stage-1:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_16.test_table2 from file:/!!ELIDED!! -INFO : Starting task [Stage-5:STATS] in serial mode -INFO : POSTHOOK: query: FROM default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_table1 +PREHOOK: Output: default@test_table2 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE test_table1 SELECT * INSERT OVERWRITE TABLE test_table2 SELECT * -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: default@src -INFO : POSTHOOK: Output: smb_mapjoin_16@test_table1 -INFO : POSTHOOK: Output: smb_mapjoin_16@test_table2 -INFO : POSTHOOK: Lineage: test_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: test_table2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-2: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-4: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query FROM default.src -INSERT OVERWRITE TABLE test_table1 SELECT * -INSERT OVERWRITE TABLE test_table2 SELECT * -No rows affected ->>> ->>> -- Mapjoin followed by a aggregation should be performed in a single MR job ->>> EXPLAIN -SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): EXPLAIN -SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): EXPLAIN -SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : PREHOOK: query: EXPLAIN -SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: EXPLAIN +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_table1 +POSTHOOK: Output: default@test_table2 +POSTHOOK: Lineage: test_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query EXPLAIN +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' Group By Operator' -' aggregations: count()' -' mode: hash' -' outputColumnNames: _col0' -' Reduce Output Operator' -' sort order: ' -' value expressions: _col0 (type: bigint)' -' Reduce Operator Tree:' -' Group By Operator' -' aggregations: count(VALUE._col0)' -' mode: mergepartial' -' outputColumnNames: _col0' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -45 rows selected ->>> SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_c1, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : PREHOOK: query: SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_16@test_table1 -INFO : PREHOOK: Input: smb_mapjoin_16@test_table2 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_16@test_table1 -INFO : POSTHOOK: Input: smb_mapjoin_16@test_table2 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key -'_c1' -'1028' -1 row selected ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+mapjoin(b)*/ count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table2 +#### A masked pattern was here #### +1028 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_2.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_2.q.out index 22a2d6a..1ea6553 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_2.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_2.q.out @@ -1,955 +1,498 @@ ->>> set hive.strict.checks.bucketing=false; -No rows affected ->>> ->>> ->>> ->>> ->>> ->>> create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_2 -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_2 -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_2 -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_2 -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_2 -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_2 -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> ->>> load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_2.smb_bucket_1 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_2 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_2.smb_bucket_2 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_2@smb_bucket_3 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_2.smb_bucket_3 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_2@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -No rows affected ->>> ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> -- SORT_QUERY_RESULTS ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'4','val_4','4','val_4' -'10','val_10','10','val_10' -2 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +4 val_4 4 val_4 +10 val_10 10 val_10 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','4','val_4' -'5','val_5','NULL','NULL' -'10','val_10','10','val_10' -5 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 4 val_4 +5 val_5 NULL NULL +10 val_10 10 val_10 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'4','val_4','4','val_4' -'10','val_10','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -6 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +4 val_4 4 val_4 +10 val_10 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 23 val_23 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','4','val_4' -'5','val_5','NULL','NULL' -'10','val_10','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -9 rows selected ->>> ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 4 val_4 +5 val_5 NULL NULL +10 val_10 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 23 val_23 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'4','val_4','4','val_4' -'10','val_10','10','val_10' -2 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +4 val_4 4 val_4 +10 val_10 10 val_10 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','4','val_4' -'5','val_5','NULL','NULL' -'10','val_10','10','val_10' -5 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 4 val_4 +5 val_5 NULL NULL +10 val_10 10 val_10 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'4','val_4','4','val_4' -'10','val_10','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -6 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +4 val_4 4 val_4 +10 val_10 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 23 val_23 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : PREHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_1 -INFO : POSTHOOK: Input: smb_mapjoin_2@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'1','val_1','NULL','NULL' -'3','val_3','NULL','NULL' -'4','val_4','4','val_4' -'5','val_5','NULL','NULL' -'10','val_10','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','23','val_23' -9 rows selected ->>> ->>> ->>> ->>> ->>> ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_1 a full outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +1 val_1 NULL NULL +3 val_3 NULL NULL +4 val_4 4 val_4 +5 val_5 NULL NULL +10 val_10 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 23 val_23 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_3.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_3.q.out index 6c9b8e4..f639ba4 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_3.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_3.q.out @@ -1,950 +1,494 @@ ->>> set hive.cbo.enable=false; -No rows affected ->>> set hive.strict.checks.bucketing=false; -No rows affected ->>> ->>> -- SORT_QUERY_RESULTS ->>> ->>> ->>> ->>> create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_3 -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_3 -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_3 -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_3 -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_3 -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_3 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_3 -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE -No rows affected ->>> ->>> load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_3.smb_bucket_1 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_2 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_3.smb_bucket_2 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 -No rows affected ->>> load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_3@smb_bucket_3 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_3.smb_bucket_3 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_3@smb_bucket_3 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 -No rows affected ->>> ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +PREHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_1 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_1 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_2 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_2 +PREHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket_3 +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -2 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +20 val_20 20 val_20 +23 val_23 23 val_23 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -'25','val_25','NULL','NULL' -'30','val_30','NULL','NULL' -4 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +20 val_20 20 val_20 +23 val_23 23 val_23 +25 val_25 NULL NULL +30 val_30 NULL NULL +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','4','val_4' -'NULL','NULL','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -6 rows selected ->>> ->>> explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +NULL NULL 4 val_4 +NULL NULL 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +20 val_20 20 val_20 +23 val_23 23 val_23 +PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','4','val_4' -'NULL','NULL','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -'25','val_25','NULL','NULL' -'30','val_30','NULL','NULL' -8 rows selected ->>> ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +NULL NULL 4 val_4 +NULL NULL 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +20 val_20 20 val_20 +23 val_23 23 val_23 +25 val_25 NULL NULL +30 val_30 NULL NULL +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Filter Operator' -' predicate: key is not null (type: boolean)' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Inner Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -37 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -2 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +20 val_20 20 val_20 +23 val_23 23 val_23 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Left Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -'25','val_25','NULL','NULL' -'30','val_30','NULL','NULL' -4 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a left outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +20 val_20 20 val_20 +23 val_23 23 val_23 +25 val_25 NULL NULL +30 val_30 NULL NULL +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Right Outer Join0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','4','val_4' -'NULL','NULL','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -6 rows selected ->>> ->>> explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: explain -select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-3:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a right outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +NULL NULL 4 val_4 +NULL NULL 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +20 val_20 20 val_20 +23 val_23 23 val_23 +PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-0 depends on stages: Stage-1' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: a' -' Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.SequenceFileInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -'' -' Stage: Stage-0' -' Fetch Operator' -' limit: -1' -' Processor Tree:' -' ListSink' -'' -34 rows selected ->>> select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : PREHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_2 -INFO : POSTHOOK: Input: smb_mapjoin_3@smb_bucket_3 -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key -'a.key','a.value','b.key','b.value' -'NULL','NULL','4','val_4' -'NULL','NULL','10','val_10' -'NULL','NULL','17','val_17' -'NULL','NULL','19','val_19' -'20','val_20','20','val_20' -'23','val_23','23','val_23' -'25','val_25','NULL','NULL' -'30','val_30','NULL','NULL' -8 rows selected ->>> ->>> ->>> ->>> ->>> ->>> !record +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_2 +PREHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +POSTHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket_2 a full outer join smb_bucket_3 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_2 +POSTHOOK: Input: default@smb_bucket_3 +#### A masked pattern was here #### +NULL NULL 4 val_4 +NULL NULL 10 val_10 +NULL NULL 17 val_17 +NULL NULL 19 val_19 +20 val_20 20 val_20 +23 val_23 23 val_23 +25 val_25 NULL NULL +30 val_30 NULL NULL diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out index b15c951..82f5804 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out @@ -1,1805 +1,1268 @@ ->>> set hive.strict.checks.bucketing=false; -No rows affected ->>> ->>> set hive.mapred.mode=nonstrict; -No rows affected ->>> set hive.exec.reducers.max = 1; -No rows affected ->>> ->>> ->>> CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: query: CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_7 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_7 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -No rows affected ->>> ->>> ->>> CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS; -INFO : Compiling commandqueryId=(!!{queryId}!!): CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: query: CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_7 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_bucket4_2 -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_7 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_bucket4_2 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS -No rows affected ->>> ->>> ->>> ->>> ->>> create table smb_join_results(k1 int, v1 string, k2 int, v2 string); -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: query: create table smb_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_7 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_join_results -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_7 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_join_results -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_join_results(k1 int, v1 string, k2 int, v2 string) -No rows affected ->>> create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string); -INFO : Compiling commandqueryId=(!!{queryId}!!): create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: query: create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_7 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_7 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) -No rows affected ->>> create table normal_join_results(k1 int, v1 string, k2 int, v2 string); -INFO : Compiling commandqueryId=(!!{queryId}!!): create table normal_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): create table normal_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : PREHOOK: type: CREATETABLE -INFO : PREHOOK: Output: database:smb_mapjoin_7 -INFO : PREHOOK: Output: smb_mapjoin_7@normal_join_results -INFO : Starting task [Stage-0:DDL] in serial mode -INFO : POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 string) -INFO : POSTHOOK: type: CREATETABLE -INFO : POSTHOOK: Output: database:smb_mapjoin_7 -INFO : POSTHOOK: Output: smb_mapjoin_7@normal_join_results -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query create table normal_join_results(k1 int, v1 string, k2 int, v2 string) -No rows affected ->>> ->>> load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 -INFO : PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_bucket4_1 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 -No rows affected ->>> load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1; -INFO : Compiling commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:null, properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 -INFO : PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 -INFO : PREHOOK: type: LOAD -INFO : PREHOOK: Input: file:/!!ELIDED!! -INFO : PREHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_bucket4_1 from file:/!!ELIDED!! -INFO : Starting task [Stage-1:STATS] in serial mode -INFO : POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 -INFO : POSTHOOK: type: LOAD -INFO : POSTHOOK: Input: file:/!!ELIDED!! -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_bucket4_1 -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 -No rows affected ->>> ->>> insert overwrite table smb_bucket4_2 -select * from default.src; -INFO : Compiling commandqueryId=(!!{queryId}!!): insert overwrite table smb_bucket4_2 -select * from default.src -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_col0, type:int, comment:null), FieldSchema(name:_col1, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): insert overwrite table smb_bucket4_2 -select * from default.src -INFO : PREHOOK: query: insert overwrite table smb_bucket4_2 -select * from default.src -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: default@src -INFO : PREHOOK: Output: smb_mapjoin_7@smb_bucket4_2 -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_bucket4_2 from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: insert overwrite table smb_bucket4_2 -select * from default.src -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: default@src -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_bucket4_2 -INFO : POSTHOOK: Lineage: smb_bucket4_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -INFO : POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query insert overwrite table smb_bucket4_2 -select * from default.src -No rows affected ->>> ->>> set hive.optimize.bucketmapjoin = true; -No rows affected ->>> set hive.optimize.bucketmapjoin.sortedmerge = true; -No rows affected ->>> set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; -No rows affected ->>> set hive.cbo.enable=false; -No rows affected ->>> insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results_empty_bigtable +PREHOOK: query: CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket4_1 +POSTHOOK: query: CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket4_1 +PREHOOK: query: CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_bucket4_2 +POSTHOOK: query: CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_bucket4_2 +PREHOOK: query: create table smb_join_results(k1 int, v1 string, k2 int, v2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_join_results +POSTHOOK: query: create table smb_join_results(k1 int, v1 string, k2 int, v2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_join_results +PREHOOK: query: create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_join_results_empty_bigtable +POSTHOOK: query: create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_join_results_empty_bigtable +PREHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@normal_join_results +POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@normal_join_results +PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_bucket4_1 +PREHOOK: query: insert overwrite table smb_bucket4_2 +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@smb_bucket4_2 +POSTHOOK: query: insert overwrite table smb_bucket4_2 +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@smb_bucket4_2 +POSTHOOK: Lineage: smb_bucket4_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table smb_join_results_empty_bigtable select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results_empty_bigtable +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket4_1 +PREHOOK: Input: default@smb_bucket4_2 +PREHOOK: Output: default@smb_join_results_empty_bigtable +POSTHOOK: query: insert overwrite table smb_join_results_empty_bigtable select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: query: insert overwrite table smb_join_results_empty_bigtable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket4_1 +POSTHOOK: Input: default@smb_bucket4_2 +POSTHOOK: Output: default@smb_join_results_empty_bigtable +POSTHOOK: Lineage: smb_join_results_empty_bigtable.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: insert overwrite table smb_join_results_empty_bigtable select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 3 -INFO : Launching Job 1 out of 3 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-7:CONDITIONAL] in serial mode -INFO : Stage-4 is filtered out by condition resolver. -INFO : Stage-3 is selected by condition resolver. -INFO : Stage-5 is filtered out by condition resolver. -INFO : Launching Job 3 out of 3 -INFO : Starting task [Stage-3:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_join_results_empty_bigtable from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: insert overwrite table smb_join_results_empty_bigtable +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket4_1 +PREHOOK: Input: default@smb_bucket4_2 +PREHOOK: Output: default@smb_join_results_empty_bigtable +POSTHOOK: query: insert overwrite table smb_join_results_empty_bigtable select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-3: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -No rows affected ->>> ->>> insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: query: insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 3 -INFO : Launching Job 1 out of 3 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-7:CONDITIONAL] in serial mode -INFO : Stage-4 is filtered out by condition resolver. -INFO : Stage-3 is selected by condition resolver. -INFO : Stage-5 is filtered out by condition resolver. -INFO : Launching Job 3 out of 3 -INFO : Starting task [Stage-3:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_join_results_empty_bigtable from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results_empty_bigtable.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-3: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query insert overwrite table smb_join_results_empty_bigtable -select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -No rows affected ->>> ->>> select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2; -INFO : Compiling commandqueryId=(!!{queryId}!!): select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:smb_join_results_empty_bigtable.k1, type:int, comment:null), FieldSchema(name:smb_join_results_empty_bigtable.v1, type:string, comment:null), FieldSchema(name:smb_join_results_empty_bigtable.k2, type:int, comment:null), FieldSchema(name:smb_join_results_empty_bigtable.v2, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 -INFO : PREHOOK: query: select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 -'smb_join_results_empty_bigtable.k1','smb_join_results_empty_bigtable.v1','smb_join_results_empty_bigtable.k2','smb_join_results_empty_bigtable.v2' -'NULL','NULL','0','val_0' -'NULL','NULL','0','val_0' -'NULL','NULL','0','val_0' -'NULL','NULL','2','val_2' -'NULL','NULL','4','val_4' -'NULL','NULL','5','val_5' -'NULL','NULL','5','val_5' -'NULL','NULL','5','val_5' -'NULL','NULL','8','val_8' -'NULL','NULL','9','val_9' -'NULL','NULL','10','val_10' -'NULL','NULL','11','val_11' -'NULL','NULL','12','val_12' -'NULL','NULL','12','val_12' -'NULL','NULL','15','val_15' -'NULL','NULL','15','val_15' -'NULL','NULL','17','val_17' -'NULL','NULL','18','val_18' -'NULL','NULL','18','val_18' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','24','val_24' -'NULL','NULL','24','val_24' -'NULL','NULL','26','val_26' -'NULL','NULL','26','val_26' -'NULL','NULL','27','val_27' -'NULL','NULL','28','val_28' -'NULL','NULL','30','val_30' -'NULL','NULL','33','val_33' -'NULL','NULL','34','val_34' -'NULL','NULL','35','val_35' -'NULL','NULL','35','val_35' -'NULL','NULL','35','val_35' -'NULL','NULL','37','val_37' -'NULL','NULL','37','val_37' -'NULL','NULL','41','val_41' -'NULL','NULL','42','val_42' -'NULL','NULL','42','val_42' -'NULL','NULL','43','val_43' -'NULL','NULL','44','val_44' -'NULL','NULL','47','val_47' -'NULL','NULL','51','val_51' -'NULL','NULL','51','val_51' -'NULL','NULL','53','val_53' -'NULL','NULL','54','val_54' -'NULL','NULL','57','val_57' -'NULL','NULL','58','val_58' -'NULL','NULL','58','val_58' -'NULL','NULL','64','val_64' -'NULL','NULL','65','val_65' -'NULL','NULL','66','val_66' -'NULL','NULL','67','val_67' -'NULL','NULL','67','val_67' -'NULL','NULL','69','val_69' -'NULL','NULL','70','val_70' -'NULL','NULL','70','val_70' -'NULL','NULL','70','val_70' -'NULL','NULL','72','val_72' -'NULL','NULL','72','val_72' -'NULL','NULL','74','val_74' -'NULL','NULL','76','val_76' -'NULL','NULL','76','val_76' -'NULL','NULL','77','val_77' -'NULL','NULL','78','val_78' -'NULL','NULL','80','val_80' -'NULL','NULL','82','val_82' -'NULL','NULL','83','val_83' -'NULL','NULL','83','val_83' -'NULL','NULL','84','val_84' -'NULL','NULL','84','val_84' -'NULL','NULL','85','val_85' -'NULL','NULL','86','val_86' -'NULL','NULL','87','val_87' -'NULL','NULL','90','val_90' -'NULL','NULL','90','val_90' -'NULL','NULL','90','val_90' -'NULL','NULL','92','val_92' -'NULL','NULL','95','val_95' -'NULL','NULL','95','val_95' -'NULL','NULL','96','val_96' -'NULL','NULL','97','val_97' -'NULL','NULL','97','val_97' -'NULL','NULL','98','val_98' -'NULL','NULL','98','val_98' -'NULL','NULL','100','val_100' -'NULL','NULL','100','val_100' -'NULL','NULL','103','val_103' -'NULL','NULL','103','val_103' -'NULL','NULL','104','val_104' -'NULL','NULL','104','val_104' -'NULL','NULL','105','val_105' -'NULL','NULL','111','val_111' -'NULL','NULL','113','val_113' -'NULL','NULL','113','val_113' -'NULL','NULL','114','val_114' -'NULL','NULL','116','val_116' -'NULL','NULL','118','val_118' -'NULL','NULL','118','val_118' -'NULL','NULL','119','val_119' -'NULL','NULL','119','val_119' -'NULL','NULL','119','val_119' -'NULL','NULL','120','val_120' -'NULL','NULL','120','val_120' -'NULL','NULL','125','val_125' -'NULL','NULL','125','val_125' -'NULL','NULL','126','val_126' -'NULL','NULL','128','val_128' -'NULL','NULL','128','val_128' -'NULL','NULL','128','val_128' -'NULL','NULL','129','val_129' -'NULL','NULL','129','val_129' -'NULL','NULL','131','val_131' -'NULL','NULL','133','val_133' -'NULL','NULL','134','val_134' -'NULL','NULL','134','val_134' -'NULL','NULL','136','val_136' -'NULL','NULL','137','val_137' -'NULL','NULL','137','val_137' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','143','val_143' -'NULL','NULL','145','val_145' -'NULL','NULL','146','val_146' -'NULL','NULL','146','val_146' -'NULL','NULL','149','val_149' -'NULL','NULL','149','val_149' -'NULL','NULL','150','val_150' -'NULL','NULL','152','val_152' -'NULL','NULL','152','val_152' -'NULL','NULL','153','val_153' -'NULL','NULL','155','val_155' -'NULL','NULL','156','val_156' -'NULL','NULL','157','val_157' -'NULL','NULL','158','val_158' -'NULL','NULL','160','val_160' -'NULL','NULL','162','val_162' -'NULL','NULL','163','val_163' -'NULL','NULL','164','val_164' -'NULL','NULL','164','val_164' -'NULL','NULL','165','val_165' -'NULL','NULL','165','val_165' -'NULL','NULL','166','val_166' -'NULL','NULL','167','val_167' -'NULL','NULL','167','val_167' -'NULL','NULL','167','val_167' -'NULL','NULL','168','val_168' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','170','val_170' -'NULL','NULL','172','val_172' -'NULL','NULL','172','val_172' -'NULL','NULL','174','val_174' -'NULL','NULL','174','val_174' -'NULL','NULL','175','val_175' -'NULL','NULL','175','val_175' -'NULL','NULL','176','val_176' -'NULL','NULL','176','val_176' -'NULL','NULL','177','val_177' -'NULL','NULL','178','val_178' -'NULL','NULL','179','val_179' -'NULL','NULL','179','val_179' -'NULL','NULL','180','val_180' -'NULL','NULL','181','val_181' -'NULL','NULL','183','val_183' -'NULL','NULL','186','val_186' -'NULL','NULL','187','val_187' -'NULL','NULL','187','val_187' -'NULL','NULL','187','val_187' -'NULL','NULL','189','val_189' -'NULL','NULL','190','val_190' -'NULL','NULL','191','val_191' -'NULL','NULL','191','val_191' -'NULL','NULL','192','val_192' -'NULL','NULL','193','val_193' -'NULL','NULL','193','val_193' -'NULL','NULL','193','val_193' -'NULL','NULL','194','val_194' -'NULL','NULL','195','val_195' -'NULL','NULL','195','val_195' -'NULL','NULL','196','val_196' -'NULL','NULL','197','val_197' -'NULL','NULL','197','val_197' -'NULL','NULL','199','val_199' -'NULL','NULL','199','val_199' -'NULL','NULL','199','val_199' -'NULL','NULL','200','val_200' -'NULL','NULL','200','val_200' -'NULL','NULL','201','val_201' -'NULL','NULL','202','val_202' -'NULL','NULL','203','val_203' -'NULL','NULL','203','val_203' -'NULL','NULL','205','val_205' -'NULL','NULL','205','val_205' -'NULL','NULL','207','val_207' -'NULL','NULL','207','val_207' -'NULL','NULL','208','val_208' -'NULL','NULL','208','val_208' -'NULL','NULL','208','val_208' -'NULL','NULL','209','val_209' -'NULL','NULL','209','val_209' -'NULL','NULL','213','val_213' -'NULL','NULL','213','val_213' -'NULL','NULL','214','val_214' -'NULL','NULL','216','val_216' -'NULL','NULL','216','val_216' -'NULL','NULL','217','val_217' -'NULL','NULL','217','val_217' -'NULL','NULL','218','val_218' -'NULL','NULL','219','val_219' -'NULL','NULL','219','val_219' -'NULL','NULL','221','val_221' -'NULL','NULL','221','val_221' -'NULL','NULL','222','val_222' -'NULL','NULL','223','val_223' -'NULL','NULL','223','val_223' -'NULL','NULL','224','val_224' -'NULL','NULL','224','val_224' -'NULL','NULL','226','val_226' -'NULL','NULL','228','val_228' -'NULL','NULL','229','val_229' -'NULL','NULL','229','val_229' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','233','val_233' -'NULL','NULL','233','val_233' -'NULL','NULL','235','val_235' -'NULL','NULL','237','val_237' -'NULL','NULL','237','val_237' -'NULL','NULL','238','val_238' -'NULL','NULL','238','val_238' -'NULL','NULL','239','val_239' -'NULL','NULL','239','val_239' -'NULL','NULL','241','val_241' -'NULL','NULL','242','val_242' -'NULL','NULL','242','val_242' -'NULL','NULL','244','val_244' -'NULL','NULL','247','val_247' -'NULL','NULL','248','val_248' -'NULL','NULL','249','val_249' -'NULL','NULL','252','val_252' -'NULL','NULL','255','val_255' -'NULL','NULL','255','val_255' -'NULL','NULL','256','val_256' -'NULL','NULL','256','val_256' -'NULL','NULL','257','val_257' -'NULL','NULL','258','val_258' -'NULL','NULL','260','val_260' -'NULL','NULL','262','val_262' -'NULL','NULL','263','val_263' -'NULL','NULL','265','val_265' -'NULL','NULL','265','val_265' -'NULL','NULL','266','val_266' -'NULL','NULL','272','val_272' -'NULL','NULL','272','val_272' -'NULL','NULL','273','val_273' -'NULL','NULL','273','val_273' -'NULL','NULL','273','val_273' -'NULL','NULL','274','val_274' -'NULL','NULL','275','val_275' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','278','val_278' -'NULL','NULL','278','val_278' -'NULL','NULL','280','val_280' -'NULL','NULL','280','val_280' -'NULL','NULL','281','val_281' -'NULL','NULL','281','val_281' -'NULL','NULL','282','val_282' -'NULL','NULL','282','val_282' -'NULL','NULL','283','val_283' -'NULL','NULL','284','val_284' -'NULL','NULL','285','val_285' -'NULL','NULL','286','val_286' -'NULL','NULL','287','val_287' -'NULL','NULL','288','val_288' -'NULL','NULL','288','val_288' -'NULL','NULL','289','val_289' -'NULL','NULL','291','val_291' -'NULL','NULL','292','val_292' -'NULL','NULL','296','val_296' -'NULL','NULL','298','val_298' -'NULL','NULL','298','val_298' -'NULL','NULL','298','val_298' -'NULL','NULL','302','val_302' -'NULL','NULL','305','val_305' -'NULL','NULL','306','val_306' -'NULL','NULL','307','val_307' -'NULL','NULL','307','val_307' -'NULL','NULL','308','val_308' -'NULL','NULL','309','val_309' -'NULL','NULL','309','val_309' -'NULL','NULL','310','val_310' -'NULL','NULL','311','val_311' -'NULL','NULL','311','val_311' -'NULL','NULL','311','val_311' -'NULL','NULL','315','val_315' -'NULL','NULL','316','val_316' -'NULL','NULL','316','val_316' -'NULL','NULL','316','val_316' -'NULL','NULL','317','val_317' -'NULL','NULL','317','val_317' -'NULL','NULL','318','val_318' -'NULL','NULL','318','val_318' -'NULL','NULL','318','val_318' -'NULL','NULL','321','val_321' -'NULL','NULL','321','val_321' -'NULL','NULL','322','val_322' -'NULL','NULL','322','val_322' -'NULL','NULL','323','val_323' -'NULL','NULL','325','val_325' -'NULL','NULL','325','val_325' -'NULL','NULL','327','val_327' -'NULL','NULL','327','val_327' -'NULL','NULL','327','val_327' -'NULL','NULL','331','val_331' -'NULL','NULL','331','val_331' -'NULL','NULL','332','val_332' -'NULL','NULL','333','val_333' -'NULL','NULL','333','val_333' -'NULL','NULL','335','val_335' -'NULL','NULL','336','val_336' -'NULL','NULL','338','val_338' -'NULL','NULL','339','val_339' -'NULL','NULL','341','val_341' -'NULL','NULL','342','val_342' -'NULL','NULL','342','val_342' -'NULL','NULL','344','val_344' -'NULL','NULL','344','val_344' -'NULL','NULL','345','val_345' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','351','val_351' -'NULL','NULL','353','val_353' -'NULL','NULL','353','val_353' -'NULL','NULL','356','val_356' -'NULL','NULL','360','val_360' -'NULL','NULL','362','val_362' -'NULL','NULL','364','val_364' -'NULL','NULL','365','val_365' -'NULL','NULL','366','val_366' -'NULL','NULL','367','val_367' -'NULL','NULL','367','val_367' -'NULL','NULL','368','val_368' -'NULL','NULL','369','val_369' -'NULL','NULL','369','val_369' -'NULL','NULL','369','val_369' -'NULL','NULL','373','val_373' -'NULL','NULL','374','val_374' -'NULL','NULL','375','val_375' -'NULL','NULL','377','val_377' -'NULL','NULL','378','val_378' -'NULL','NULL','379','val_379' -'NULL','NULL','382','val_382' -'NULL','NULL','382','val_382' -'NULL','NULL','384','val_384' -'NULL','NULL','384','val_384' -'NULL','NULL','384','val_384' -'NULL','NULL','386','val_386' -'NULL','NULL','389','val_389' -'NULL','NULL','392','val_392' -'NULL','NULL','393','val_393' -'NULL','NULL','394','val_394' -'NULL','NULL','395','val_395' -'NULL','NULL','395','val_395' -'NULL','NULL','396','val_396' -'NULL','NULL','396','val_396' -'NULL','NULL','396','val_396' -'NULL','NULL','397','val_397' -'NULL','NULL','397','val_397' -'NULL','NULL','399','val_399' -'NULL','NULL','399','val_399' -'NULL','NULL','400','val_400' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','402','val_402' -'NULL','NULL','403','val_403' -'NULL','NULL','403','val_403' -'NULL','NULL','403','val_403' -'NULL','NULL','404','val_404' -'NULL','NULL','404','val_404' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','407','val_407' -'NULL','NULL','409','val_409' -'NULL','NULL','409','val_409' -'NULL','NULL','409','val_409' -'NULL','NULL','411','val_411' -'NULL','NULL','413','val_413' -'NULL','NULL','413','val_413' -'NULL','NULL','414','val_414' -'NULL','NULL','414','val_414' -'NULL','NULL','417','val_417' -'NULL','NULL','417','val_417' -'NULL','NULL','417','val_417' -'NULL','NULL','418','val_418' -'NULL','NULL','419','val_419' -'NULL','NULL','421','val_421' -'NULL','NULL','424','val_424' -'NULL','NULL','424','val_424' -'NULL','NULL','427','val_427' -'NULL','NULL','429','val_429' -'NULL','NULL','429','val_429' -'NULL','NULL','430','val_430' -'NULL','NULL','430','val_430' -'NULL','NULL','430','val_430' -'NULL','NULL','431','val_431' -'NULL','NULL','431','val_431' -'NULL','NULL','431','val_431' -'NULL','NULL','432','val_432' -'NULL','NULL','435','val_435' -'NULL','NULL','436','val_436' -'NULL','NULL','437','val_437' -'NULL','NULL','438','val_438' -'NULL','NULL','438','val_438' -'NULL','NULL','438','val_438' -'NULL','NULL','439','val_439' -'NULL','NULL','439','val_439' -'NULL','NULL','443','val_443' -'NULL','NULL','444','val_444' -'NULL','NULL','446','val_446' -'NULL','NULL','448','val_448' -'NULL','NULL','449','val_449' -'NULL','NULL','452','val_452' -'NULL','NULL','453','val_453' -'NULL','NULL','454','val_454' -'NULL','NULL','454','val_454' -'NULL','NULL','454','val_454' -'NULL','NULL','455','val_455' -'NULL','NULL','457','val_457' -'NULL','NULL','458','val_458' -'NULL','NULL','458','val_458' -'NULL','NULL','459','val_459' -'NULL','NULL','459','val_459' -'NULL','NULL','460','val_460' -'NULL','NULL','462','val_462' -'NULL','NULL','462','val_462' -'NULL','NULL','463','val_463' -'NULL','NULL','463','val_463' -'NULL','NULL','466','val_466' -'NULL','NULL','466','val_466' -'NULL','NULL','466','val_466' -'NULL','NULL','467','val_467' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','470','val_470' -'NULL','NULL','472','val_472' -'NULL','NULL','475','val_475' -'NULL','NULL','477','val_477' -'NULL','NULL','478','val_478' -'NULL','NULL','478','val_478' -'NULL','NULL','479','val_479' -'NULL','NULL','480','val_480' -'NULL','NULL','480','val_480' -'NULL','NULL','480','val_480' -'NULL','NULL','481','val_481' -'NULL','NULL','482','val_482' -'NULL','NULL','483','val_483' -'NULL','NULL','484','val_484' -'NULL','NULL','485','val_485' -'NULL','NULL','487','val_487' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','490','val_490' -'NULL','NULL','491','val_491' -'NULL','NULL','492','val_492' -'NULL','NULL','492','val_492' -'NULL','NULL','493','val_493' -'NULL','NULL','494','val_494' -'NULL','NULL','495','val_495' -'NULL','NULL','496','val_496' -'NULL','NULL','497','val_497' -'NULL','NULL','498','val_498' -'NULL','NULL','498','val_498' -'NULL','NULL','498','val_498' -500 rows selected ->>> ->>> explain -insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): explain -insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:Explain, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): explain -insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: query: explain -insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : Starting task [Stage-9:EXPLAIN] in serial mode -INFO : POSTHOOK: query: explain +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket4_1 +POSTHOOK: Input: default@smb_bucket4_2 +POSTHOOK: Output: default@smb_join_results_empty_bigtable +POSTHOOK: Lineage: smb_join_results_empty_bigtable.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: smb_join_results_empty_bigtable.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_join_results_empty_bigtable +#### A masked pattern was here #### +POSTHOOK: query: select * from smb_join_results_empty_bigtable order by k1, v1, k2, v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_join_results_empty_bigtable +#### A masked pattern was here #### +NULL NULL 0 val_0 +NULL NULL 0 val_0 +NULL NULL 0 val_0 +NULL NULL 2 val_2 +NULL NULL 4 val_4 +NULL NULL 5 val_5 +NULL NULL 5 val_5 +NULL NULL 5 val_5 +NULL NULL 8 val_8 +NULL NULL 9 val_9 +NULL NULL 10 val_10 +NULL NULL 11 val_11 +NULL NULL 12 val_12 +NULL NULL 12 val_12 +NULL NULL 15 val_15 +NULL NULL 15 val_15 +NULL NULL 17 val_17 +NULL NULL 18 val_18 +NULL NULL 18 val_18 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 24 val_24 +NULL NULL 24 val_24 +NULL NULL 26 val_26 +NULL NULL 26 val_26 +NULL NULL 27 val_27 +NULL NULL 28 val_28 +NULL NULL 30 val_30 +NULL NULL 33 val_33 +NULL NULL 34 val_34 +NULL NULL 35 val_35 +NULL NULL 35 val_35 +NULL NULL 35 val_35 +NULL NULL 37 val_37 +NULL NULL 37 val_37 +NULL NULL 41 val_41 +NULL NULL 42 val_42 +NULL NULL 42 val_42 +NULL NULL 43 val_43 +NULL NULL 44 val_44 +NULL NULL 47 val_47 +NULL NULL 51 val_51 +NULL NULL 51 val_51 +NULL NULL 53 val_53 +NULL NULL 54 val_54 +NULL NULL 57 val_57 +NULL NULL 58 val_58 +NULL NULL 58 val_58 +NULL NULL 64 val_64 +NULL NULL 65 val_65 +NULL NULL 66 val_66 +NULL NULL 67 val_67 +NULL NULL 67 val_67 +NULL NULL 69 val_69 +NULL NULL 70 val_70 +NULL NULL 70 val_70 +NULL NULL 70 val_70 +NULL NULL 72 val_72 +NULL NULL 72 val_72 +NULL NULL 74 val_74 +NULL NULL 76 val_76 +NULL NULL 76 val_76 +NULL NULL 77 val_77 +NULL NULL 78 val_78 +NULL NULL 80 val_80 +NULL NULL 82 val_82 +NULL NULL 83 val_83 +NULL NULL 83 val_83 +NULL NULL 84 val_84 +NULL NULL 84 val_84 +NULL NULL 85 val_85 +NULL NULL 86 val_86 +NULL NULL 87 val_87 +NULL NULL 90 val_90 +NULL NULL 90 val_90 +NULL NULL 90 val_90 +NULL NULL 92 val_92 +NULL NULL 95 val_95 +NULL NULL 95 val_95 +NULL NULL 96 val_96 +NULL NULL 97 val_97 +NULL NULL 97 val_97 +NULL NULL 98 val_98 +NULL NULL 98 val_98 +NULL NULL 100 val_100 +NULL NULL 100 val_100 +NULL NULL 103 val_103 +NULL NULL 103 val_103 +NULL NULL 104 val_104 +NULL NULL 104 val_104 +NULL NULL 105 val_105 +NULL NULL 111 val_111 +NULL NULL 113 val_113 +NULL NULL 113 val_113 +NULL NULL 114 val_114 +NULL NULL 116 val_116 +NULL NULL 118 val_118 +NULL NULL 118 val_118 +NULL NULL 119 val_119 +NULL NULL 119 val_119 +NULL NULL 119 val_119 +NULL NULL 120 val_120 +NULL NULL 120 val_120 +NULL NULL 125 val_125 +NULL NULL 125 val_125 +NULL NULL 126 val_126 +NULL NULL 128 val_128 +NULL NULL 128 val_128 +NULL NULL 128 val_128 +NULL NULL 129 val_129 +NULL NULL 129 val_129 +NULL NULL 131 val_131 +NULL NULL 133 val_133 +NULL NULL 134 val_134 +NULL NULL 134 val_134 +NULL NULL 136 val_136 +NULL NULL 137 val_137 +NULL NULL 137 val_137 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 143 val_143 +NULL NULL 145 val_145 +NULL NULL 146 val_146 +NULL NULL 146 val_146 +NULL NULL 149 val_149 +NULL NULL 149 val_149 +NULL NULL 150 val_150 +NULL NULL 152 val_152 +NULL NULL 152 val_152 +NULL NULL 153 val_153 +NULL NULL 155 val_155 +NULL NULL 156 val_156 +NULL NULL 157 val_157 +NULL NULL 158 val_158 +NULL NULL 160 val_160 +NULL NULL 162 val_162 +NULL NULL 163 val_163 +NULL NULL 164 val_164 +NULL NULL 164 val_164 +NULL NULL 165 val_165 +NULL NULL 165 val_165 +NULL NULL 166 val_166 +NULL NULL 167 val_167 +NULL NULL 167 val_167 +NULL NULL 167 val_167 +NULL NULL 168 val_168 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 170 val_170 +NULL NULL 172 val_172 +NULL NULL 172 val_172 +NULL NULL 174 val_174 +NULL NULL 174 val_174 +NULL NULL 175 val_175 +NULL NULL 175 val_175 +NULL NULL 176 val_176 +NULL NULL 176 val_176 +NULL NULL 177 val_177 +NULL NULL 178 val_178 +NULL NULL 179 val_179 +NULL NULL 179 val_179 +NULL NULL 180 val_180 +NULL NULL 181 val_181 +NULL NULL 183 val_183 +NULL NULL 186 val_186 +NULL NULL 187 val_187 +NULL NULL 187 val_187 +NULL NULL 187 val_187 +NULL NULL 189 val_189 +NULL NULL 190 val_190 +NULL NULL 191 val_191 +NULL NULL 191 val_191 +NULL NULL 192 val_192 +NULL NULL 193 val_193 +NULL NULL 193 val_193 +NULL NULL 193 val_193 +NULL NULL 194 val_194 +NULL NULL 195 val_195 +NULL NULL 195 val_195 +NULL NULL 196 val_196 +NULL NULL 197 val_197 +NULL NULL 197 val_197 +NULL NULL 199 val_199 +NULL NULL 199 val_199 +NULL NULL 199 val_199 +NULL NULL 200 val_200 +NULL NULL 200 val_200 +NULL NULL 201 val_201 +NULL NULL 202 val_202 +NULL NULL 203 val_203 +NULL NULL 203 val_203 +NULL NULL 205 val_205 +NULL NULL 205 val_205 +NULL NULL 207 val_207 +NULL NULL 207 val_207 +NULL NULL 208 val_208 +NULL NULL 208 val_208 +NULL NULL 208 val_208 +NULL NULL 209 val_209 +NULL NULL 209 val_209 +NULL NULL 213 val_213 +NULL NULL 213 val_213 +NULL NULL 214 val_214 +NULL NULL 216 val_216 +NULL NULL 216 val_216 +NULL NULL 217 val_217 +NULL NULL 217 val_217 +NULL NULL 218 val_218 +NULL NULL 219 val_219 +NULL NULL 219 val_219 +NULL NULL 221 val_221 +NULL NULL 221 val_221 +NULL NULL 222 val_222 +NULL NULL 223 val_223 +NULL NULL 223 val_223 +NULL NULL 224 val_224 +NULL NULL 224 val_224 +NULL NULL 226 val_226 +NULL NULL 228 val_228 +NULL NULL 229 val_229 +NULL NULL 229 val_229 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 233 val_233 +NULL NULL 233 val_233 +NULL NULL 235 val_235 +NULL NULL 237 val_237 +NULL NULL 237 val_237 +NULL NULL 238 val_238 +NULL NULL 238 val_238 +NULL NULL 239 val_239 +NULL NULL 239 val_239 +NULL NULL 241 val_241 +NULL NULL 242 val_242 +NULL NULL 242 val_242 +NULL NULL 244 val_244 +NULL NULL 247 val_247 +NULL NULL 248 val_248 +NULL NULL 249 val_249 +NULL NULL 252 val_252 +NULL NULL 255 val_255 +NULL NULL 255 val_255 +NULL NULL 256 val_256 +NULL NULL 256 val_256 +NULL NULL 257 val_257 +NULL NULL 258 val_258 +NULL NULL 260 val_260 +NULL NULL 262 val_262 +NULL NULL 263 val_263 +NULL NULL 265 val_265 +NULL NULL 265 val_265 +NULL NULL 266 val_266 +NULL NULL 272 val_272 +NULL NULL 272 val_272 +NULL NULL 273 val_273 +NULL NULL 273 val_273 +NULL NULL 273 val_273 +NULL NULL 274 val_274 +NULL NULL 275 val_275 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 278 val_278 +NULL NULL 278 val_278 +NULL NULL 280 val_280 +NULL NULL 280 val_280 +NULL NULL 281 val_281 +NULL NULL 281 val_281 +NULL NULL 282 val_282 +NULL NULL 282 val_282 +NULL NULL 283 val_283 +NULL NULL 284 val_284 +NULL NULL 285 val_285 +NULL NULL 286 val_286 +NULL NULL 287 val_287 +NULL NULL 288 val_288 +NULL NULL 288 val_288 +NULL NULL 289 val_289 +NULL NULL 291 val_291 +NULL NULL 292 val_292 +NULL NULL 296 val_296 +NULL NULL 298 val_298 +NULL NULL 298 val_298 +NULL NULL 298 val_298 +NULL NULL 302 val_302 +NULL NULL 305 val_305 +NULL NULL 306 val_306 +NULL NULL 307 val_307 +NULL NULL 307 val_307 +NULL NULL 308 val_308 +NULL NULL 309 val_309 +NULL NULL 309 val_309 +NULL NULL 310 val_310 +NULL NULL 311 val_311 +NULL NULL 311 val_311 +NULL NULL 311 val_311 +NULL NULL 315 val_315 +NULL NULL 316 val_316 +NULL NULL 316 val_316 +NULL NULL 316 val_316 +NULL NULL 317 val_317 +NULL NULL 317 val_317 +NULL NULL 318 val_318 +NULL NULL 318 val_318 +NULL NULL 318 val_318 +NULL NULL 321 val_321 +NULL NULL 321 val_321 +NULL NULL 322 val_322 +NULL NULL 322 val_322 +NULL NULL 323 val_323 +NULL NULL 325 val_325 +NULL NULL 325 val_325 +NULL NULL 327 val_327 +NULL NULL 327 val_327 +NULL NULL 327 val_327 +NULL NULL 331 val_331 +NULL NULL 331 val_331 +NULL NULL 332 val_332 +NULL NULL 333 val_333 +NULL NULL 333 val_333 +NULL NULL 335 val_335 +NULL NULL 336 val_336 +NULL NULL 338 val_338 +NULL NULL 339 val_339 +NULL NULL 341 val_341 +NULL NULL 342 val_342 +NULL NULL 342 val_342 +NULL NULL 344 val_344 +NULL NULL 344 val_344 +NULL NULL 345 val_345 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 351 val_351 +NULL NULL 353 val_353 +NULL NULL 353 val_353 +NULL NULL 356 val_356 +NULL NULL 360 val_360 +NULL NULL 362 val_362 +NULL NULL 364 val_364 +NULL NULL 365 val_365 +NULL NULL 366 val_366 +NULL NULL 367 val_367 +NULL NULL 367 val_367 +NULL NULL 368 val_368 +NULL NULL 369 val_369 +NULL NULL 369 val_369 +NULL NULL 369 val_369 +NULL NULL 373 val_373 +NULL NULL 374 val_374 +NULL NULL 375 val_375 +NULL NULL 377 val_377 +NULL NULL 378 val_378 +NULL NULL 379 val_379 +NULL NULL 382 val_382 +NULL NULL 382 val_382 +NULL NULL 384 val_384 +NULL NULL 384 val_384 +NULL NULL 384 val_384 +NULL NULL 386 val_386 +NULL NULL 389 val_389 +NULL NULL 392 val_392 +NULL NULL 393 val_393 +NULL NULL 394 val_394 +NULL NULL 395 val_395 +NULL NULL 395 val_395 +NULL NULL 396 val_396 +NULL NULL 396 val_396 +NULL NULL 396 val_396 +NULL NULL 397 val_397 +NULL NULL 397 val_397 +NULL NULL 399 val_399 +NULL NULL 399 val_399 +NULL NULL 400 val_400 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 402 val_402 +NULL NULL 403 val_403 +NULL NULL 403 val_403 +NULL NULL 403 val_403 +NULL NULL 404 val_404 +NULL NULL 404 val_404 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 407 val_407 +NULL NULL 409 val_409 +NULL NULL 409 val_409 +NULL NULL 409 val_409 +NULL NULL 411 val_411 +NULL NULL 413 val_413 +NULL NULL 413 val_413 +NULL NULL 414 val_414 +NULL NULL 414 val_414 +NULL NULL 417 val_417 +NULL NULL 417 val_417 +NULL NULL 417 val_417 +NULL NULL 418 val_418 +NULL NULL 419 val_419 +NULL NULL 421 val_421 +NULL NULL 424 val_424 +NULL NULL 424 val_424 +NULL NULL 427 val_427 +NULL NULL 429 val_429 +NULL NULL 429 val_429 +NULL NULL 430 val_430 +NULL NULL 430 val_430 +NULL NULL 430 val_430 +NULL NULL 431 val_431 +NULL NULL 431 val_431 +NULL NULL 431 val_431 +NULL NULL 432 val_432 +NULL NULL 435 val_435 +NULL NULL 436 val_436 +NULL NULL 437 val_437 +NULL NULL 438 val_438 +NULL NULL 438 val_438 +NULL NULL 438 val_438 +NULL NULL 439 val_439 +NULL NULL 439 val_439 +NULL NULL 443 val_443 +NULL NULL 444 val_444 +NULL NULL 446 val_446 +NULL NULL 448 val_448 +NULL NULL 449 val_449 +NULL NULL 452 val_452 +NULL NULL 453 val_453 +NULL NULL 454 val_454 +NULL NULL 454 val_454 +NULL NULL 454 val_454 +NULL NULL 455 val_455 +NULL NULL 457 val_457 +NULL NULL 458 val_458 +NULL NULL 458 val_458 +NULL NULL 459 val_459 +NULL NULL 459 val_459 +NULL NULL 460 val_460 +NULL NULL 462 val_462 +NULL NULL 462 val_462 +NULL NULL 463 val_463 +NULL NULL 463 val_463 +NULL NULL 466 val_466 +NULL NULL 466 val_466 +NULL NULL 466 val_466 +NULL NULL 467 val_467 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 470 val_470 +NULL NULL 472 val_472 +NULL NULL 475 val_475 +NULL NULL 477 val_477 +NULL NULL 478 val_478 +NULL NULL 478 val_478 +NULL NULL 479 val_479 +NULL NULL 480 val_480 +NULL NULL 480 val_480 +NULL NULL 480 val_480 +NULL NULL 481 val_481 +NULL NULL 482 val_482 +NULL NULL 483 val_483 +NULL NULL 484 val_484 +NULL NULL 485 val_485 +NULL NULL 487 val_487 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 490 val_490 +NULL NULL 491 val_491 +NULL NULL 492 val_492 +NULL NULL 492 val_492 +NULL NULL 493 val_493 +NULL NULL 494 val_494 +NULL NULL 495 val_495 +NULL NULL 496 val_496 +NULL NULL 497 val_497 +NULL NULL 498 val_498 +NULL NULL 498 val_498 +NULL NULL 498 val_498 +PREHOOK: query: explain insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query explain +PREHOOK: type: QUERY +POSTHOOK: query: explain insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -'Explain' -'STAGE DEPENDENCIES:' -' Stage-1 is a root stage' -' Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6' -' Stage-5' -' Stage-0 depends on stages: Stage-5, Stage-4, Stage-7' -' Stage-3 depends on stages: Stage-0' -' Stage-4' -' Stage-6' -' Stage-7 depends on stages: Stage-6' -'' -'STAGE PLANS:' -' Stage: Stage-1' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' alias: b' -' Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE' -' Sorted Merge Bucket Map Join Operator' -' condition map:' -' Outer Join 0 to 1' -' keys:' -' 0 key (type: int)' -' 1 key (type: int)' -' outputColumnNames: _col0, _col1, _col5, _col6' -' Select Operator' -' expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)' -' outputColumnNames: _col0, _col1, _col2, _col3' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_7.smb_join_results' -'' -' Stage: Stage-8' -' Conditional Operator' -'' -' Stage: Stage-5' -' Move Operator' -' files:' -' hdfs directory: true' -' destination: file:/!!ELIDED!! -'' -' Stage: Stage-0' -' Move Operator' -' tables:' -' replace: true' -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_7.smb_join_results' -'' -' Stage: Stage-3' -' Stats-Aggr Operator' -'' -' Stage: Stage-4' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_7.smb_join_results' -'' -' Stage: Stage-6' -' Map Reduce' -' Map Operator Tree:' -' TableScan' -' File Output Operator' -' compressed: false' -' table:' -' input format: org.apache.hadoop.mapred.TextInputFormat' -' output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -' serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -' name: smb_mapjoin_7.smb_join_results' -'' -' Stage: Stage-7' -' Move Operator' -' files:' -' hdfs directory: true' -' destination: file:/!!ELIDED!! -'' -87 rows selected ->>> ->>> insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: query: insert overwrite table smb_join_results -select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : PREHOOK: Output: smb_mapjoin_7@smb_join_results -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 3 -INFO : Launching Job 1 out of 3 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:2 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-8:CONDITIONAL] in serial mode -INFO : Stage-5 is filtered out by condition resolver. -INFO : Stage-4 is selected by condition resolver. -INFO : Stage-6 is filtered out by condition resolver. -INFO : Launching Job 3 out of 3 -INFO : Starting task [Stage-4:MAPRED] in serial mode -INFO : Number of reduce tasks is set to 0 since there's no reduce operator -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.smb_join_results from file:/!!ELIDED!! -INFO : Starting task [Stage-3:STATS] in serial mode -INFO : POSTHOOK: query: insert overwrite table smb_join_results +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 + Stage-5 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-3 depends on stages: Stage-0 + Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results + + Stage: Stage-8 + Conditional Operator + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : POSTHOOK: Output: smb_mapjoin_7@smb_join_results -INFO : POSTHOOK: Lineage: smb_join_results.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] -INFO : POSTHOOK: Lineage: smb_join_results.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Stage-Stage-4: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query insert overwrite table smb_join_results +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket4_1 +PREHOOK: Input: default@smb_bucket4_2 +PREHOOK: Output: default@smb_join_results +POSTHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -No rows affected ->>> ->>> select * from smb_join_results order by k1, v1, k2, v2; -INFO : Compiling commandqueryId=(!!{queryId}!!): select * from smb_join_results order by k1, v1, k2, v2 -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:smb_join_results.k1, type:int, comment:null), FieldSchema(name:smb_join_results.v1, type:string, comment:null), FieldSchema(name:smb_join_results.k2, type:int, comment:null), FieldSchema(name:smb_join_results.v2, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select * from smb_join_results order by k1, v1, k2, v2 -INFO : PREHOOK: query: select * from smb_join_results order by k1, v1, k2, v2 -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_join_results -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select * from smb_join_results order by k1, v1, k2, v2 -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_join_results -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select * from smb_join_results order by k1, v1, k2, v2 -'smb_join_results.k1','smb_join_results.v1','smb_join_results.k2','smb_join_results.v2' -'NULL','NULL','0','val_0' -'NULL','NULL','0','val_0' -'NULL','NULL','0','val_0' -'NULL','NULL','2','val_2' -'NULL','NULL','4','val_4' -'NULL','NULL','5','val_5' -'NULL','NULL','5','val_5' -'NULL','NULL','5','val_5' -'NULL','NULL','8','val_8' -'NULL','NULL','9','val_9' -'NULL','NULL','10','val_10' -'NULL','NULL','11','val_11' -'NULL','NULL','12','val_12' -'NULL','NULL','12','val_12' -'NULL','NULL','15','val_15' -'NULL','NULL','15','val_15' -'NULL','NULL','17','val_17' -'NULL','NULL','18','val_18' -'NULL','NULL','18','val_18' -'NULL','NULL','19','val_19' -'NULL','NULL','20','val_20' -'NULL','NULL','24','val_24' -'NULL','NULL','24','val_24' -'NULL','NULL','26','val_26' -'NULL','NULL','26','val_26' -'NULL','NULL','27','val_27' -'NULL','NULL','28','val_28' -'NULL','NULL','30','val_30' -'NULL','NULL','33','val_33' -'NULL','NULL','34','val_34' -'NULL','NULL','35','val_35' -'NULL','NULL','35','val_35' -'NULL','NULL','35','val_35' -'NULL','NULL','37','val_37' -'NULL','NULL','37','val_37' -'NULL','NULL','41','val_41' -'NULL','NULL','42','val_42' -'NULL','NULL','42','val_42' -'NULL','NULL','43','val_43' -'NULL','NULL','44','val_44' -'NULL','NULL','47','val_47' -'NULL','NULL','51','val_51' -'NULL','NULL','51','val_51' -'NULL','NULL','53','val_53' -'NULL','NULL','54','val_54' -'NULL','NULL','57','val_57' -'NULL','NULL','58','val_58' -'NULL','NULL','58','val_58' -'NULL','NULL','64','val_64' -'NULL','NULL','65','val_65' -'NULL','NULL','66','val_66' -'NULL','NULL','67','val_67' -'NULL','NULL','67','val_67' -'NULL','NULL','69','val_69' -'NULL','NULL','70','val_70' -'NULL','NULL','70','val_70' -'NULL','NULL','70','val_70' -'NULL','NULL','72','val_72' -'NULL','NULL','72','val_72' -'NULL','NULL','74','val_74' -'NULL','NULL','76','val_76' -'NULL','NULL','76','val_76' -'NULL','NULL','77','val_77' -'NULL','NULL','78','val_78' -'NULL','NULL','80','val_80' -'NULL','NULL','82','val_82' -'NULL','NULL','83','val_83' -'NULL','NULL','83','val_83' -'NULL','NULL','84','val_84' -'NULL','NULL','84','val_84' -'NULL','NULL','85','val_85' -'NULL','NULL','86','val_86' -'NULL','NULL','87','val_87' -'NULL','NULL','90','val_90' -'NULL','NULL','90','val_90' -'NULL','NULL','90','val_90' -'NULL','NULL','92','val_92' -'NULL','NULL','95','val_95' -'NULL','NULL','95','val_95' -'NULL','NULL','96','val_96' -'NULL','NULL','97','val_97' -'NULL','NULL','97','val_97' -'NULL','NULL','98','val_98' -'NULL','NULL','98','val_98' -'NULL','NULL','100','val_100' -'NULL','NULL','100','val_100' -'NULL','NULL','103','val_103' -'NULL','NULL','103','val_103' -'NULL','NULL','104','val_104' -'NULL','NULL','104','val_104' -'NULL','NULL','105','val_105' -'NULL','NULL','111','val_111' -'NULL','NULL','113','val_113' -'NULL','NULL','113','val_113' -'NULL','NULL','114','val_114' -'NULL','NULL','116','val_116' -'NULL','NULL','118','val_118' -'NULL','NULL','118','val_118' -'NULL','NULL','119','val_119' -'NULL','NULL','119','val_119' -'NULL','NULL','119','val_119' -'NULL','NULL','120','val_120' -'NULL','NULL','120','val_120' -'NULL','NULL','125','val_125' -'NULL','NULL','125','val_125' -'NULL','NULL','126','val_126' -'NULL','NULL','128','val_128' -'NULL','NULL','128','val_128' -'NULL','NULL','128','val_128' -'NULL','NULL','129','val_129' -'NULL','NULL','129','val_129' -'NULL','NULL','131','val_131' -'NULL','NULL','133','val_133' -'NULL','NULL','134','val_134' -'NULL','NULL','134','val_134' -'NULL','NULL','136','val_136' -'NULL','NULL','137','val_137' -'NULL','NULL','137','val_137' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','138','val_138' -'NULL','NULL','143','val_143' -'NULL','NULL','145','val_145' -'NULL','NULL','146','val_146' -'NULL','NULL','146','val_146' -'NULL','NULL','149','val_149' -'NULL','NULL','149','val_149' -'NULL','NULL','150','val_150' -'NULL','NULL','152','val_152' -'NULL','NULL','152','val_152' -'NULL','NULL','153','val_153' -'NULL','NULL','155','val_155' -'NULL','NULL','156','val_156' -'NULL','NULL','157','val_157' -'NULL','NULL','158','val_158' -'NULL','NULL','160','val_160' -'NULL','NULL','162','val_162' -'NULL','NULL','163','val_163' -'NULL','NULL','164','val_164' -'NULL','NULL','164','val_164' -'NULL','NULL','165','val_165' -'NULL','NULL','165','val_165' -'NULL','NULL','166','val_166' -'NULL','NULL','167','val_167' -'NULL','NULL','167','val_167' -'NULL','NULL','167','val_167' -'NULL','NULL','168','val_168' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','169','val_169' -'NULL','NULL','170','val_170' -'NULL','NULL','172','val_172' -'NULL','NULL','172','val_172' -'NULL','NULL','174','val_174' -'NULL','NULL','174','val_174' -'NULL','NULL','175','val_175' -'NULL','NULL','175','val_175' -'NULL','NULL','176','val_176' -'NULL','NULL','176','val_176' -'NULL','NULL','177','val_177' -'NULL','NULL','178','val_178' -'NULL','NULL','179','val_179' -'NULL','NULL','179','val_179' -'NULL','NULL','180','val_180' -'NULL','NULL','181','val_181' -'NULL','NULL','183','val_183' -'NULL','NULL','186','val_186' -'NULL','NULL','187','val_187' -'NULL','NULL','187','val_187' -'NULL','NULL','187','val_187' -'NULL','NULL','189','val_189' -'NULL','NULL','190','val_190' -'NULL','NULL','191','val_191' -'NULL','NULL','191','val_191' -'NULL','NULL','192','val_192' -'NULL','NULL','193','val_193' -'NULL','NULL','193','val_193' -'NULL','NULL','193','val_193' -'NULL','NULL','194','val_194' -'NULL','NULL','195','val_195' -'NULL','NULL','195','val_195' -'NULL','NULL','196','val_196' -'NULL','NULL','197','val_197' -'NULL','NULL','197','val_197' -'NULL','NULL','199','val_199' -'NULL','NULL','199','val_199' -'NULL','NULL','199','val_199' -'NULL','NULL','200','val_200' -'NULL','NULL','200','val_200' -'NULL','NULL','201','val_201' -'NULL','NULL','202','val_202' -'NULL','NULL','203','val_203' -'NULL','NULL','203','val_203' -'NULL','NULL','205','val_205' -'NULL','NULL','205','val_205' -'NULL','NULL','207','val_207' -'NULL','NULL','207','val_207' -'NULL','NULL','208','val_208' -'NULL','NULL','208','val_208' -'NULL','NULL','208','val_208' -'NULL','NULL','209','val_209' -'NULL','NULL','209','val_209' -'NULL','NULL','213','val_213' -'NULL','NULL','213','val_213' -'NULL','NULL','214','val_214' -'NULL','NULL','216','val_216' -'NULL','NULL','216','val_216' -'NULL','NULL','217','val_217' -'NULL','NULL','217','val_217' -'NULL','NULL','218','val_218' -'NULL','NULL','219','val_219' -'NULL','NULL','219','val_219' -'NULL','NULL','221','val_221' -'NULL','NULL','221','val_221' -'NULL','NULL','222','val_222' -'NULL','NULL','223','val_223' -'NULL','NULL','223','val_223' -'NULL','NULL','224','val_224' -'NULL','NULL','224','val_224' -'NULL','NULL','226','val_226' -'NULL','NULL','228','val_228' -'NULL','NULL','229','val_229' -'NULL','NULL','229','val_229' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','230','val_230' -'NULL','NULL','233','val_233' -'NULL','NULL','233','val_233' -'NULL','NULL','235','val_235' -'NULL','NULL','237','val_237' -'NULL','NULL','237','val_237' -'NULL','NULL','238','val_238' -'NULL','NULL','238','val_238' -'NULL','NULL','239','val_239' -'NULL','NULL','239','val_239' -'NULL','NULL','241','val_241' -'NULL','NULL','242','val_242' -'NULL','NULL','242','val_242' -'NULL','NULL','244','val_244' -'NULL','NULL','247','val_247' -'NULL','NULL','248','val_248' -'NULL','NULL','249','val_249' -'NULL','NULL','252','val_252' -'NULL','NULL','255','val_255' -'NULL','NULL','255','val_255' -'NULL','NULL','256','val_256' -'NULL','NULL','256','val_256' -'NULL','NULL','257','val_257' -'NULL','NULL','258','val_258' -'NULL','NULL','260','val_260' -'NULL','NULL','262','val_262' -'NULL','NULL','263','val_263' -'NULL','NULL','265','val_265' -'NULL','NULL','265','val_265' -'NULL','NULL','266','val_266' -'NULL','NULL','272','val_272' -'NULL','NULL','272','val_272' -'NULL','NULL','273','val_273' -'NULL','NULL','273','val_273' -'NULL','NULL','273','val_273' -'NULL','NULL','274','val_274' -'NULL','NULL','275','val_275' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','277','val_277' -'NULL','NULL','278','val_278' -'NULL','NULL','278','val_278' -'NULL','NULL','280','val_280' -'NULL','NULL','280','val_280' -'NULL','NULL','281','val_281' -'NULL','NULL','281','val_281' -'NULL','NULL','282','val_282' -'NULL','NULL','282','val_282' -'NULL','NULL','283','val_283' -'NULL','NULL','284','val_284' -'NULL','NULL','285','val_285' -'NULL','NULL','286','val_286' -'NULL','NULL','287','val_287' -'NULL','NULL','288','val_288' -'NULL','NULL','288','val_288' -'NULL','NULL','289','val_289' -'NULL','NULL','291','val_291' -'NULL','NULL','292','val_292' -'NULL','NULL','296','val_296' -'NULL','NULL','298','val_298' -'NULL','NULL','298','val_298' -'NULL','NULL','298','val_298' -'NULL','NULL','302','val_302' -'NULL','NULL','305','val_305' -'NULL','NULL','306','val_306' -'NULL','NULL','307','val_307' -'NULL','NULL','307','val_307' -'NULL','NULL','308','val_308' -'NULL','NULL','309','val_309' -'NULL','NULL','309','val_309' -'NULL','NULL','310','val_310' -'NULL','NULL','311','val_311' -'NULL','NULL','311','val_311' -'NULL','NULL','311','val_311' -'NULL','NULL','315','val_315' -'NULL','NULL','316','val_316' -'NULL','NULL','316','val_316' -'NULL','NULL','316','val_316' -'NULL','NULL','317','val_317' -'NULL','NULL','317','val_317' -'NULL','NULL','318','val_318' -'NULL','NULL','318','val_318' -'NULL','NULL','318','val_318' -'NULL','NULL','321','val_321' -'NULL','NULL','321','val_321' -'NULL','NULL','322','val_322' -'NULL','NULL','322','val_322' -'NULL','NULL','323','val_323' -'NULL','NULL','325','val_325' -'NULL','NULL','325','val_325' -'NULL','NULL','327','val_327' -'NULL','NULL','327','val_327' -'NULL','NULL','327','val_327' -'NULL','NULL','331','val_331' -'NULL','NULL','331','val_331' -'NULL','NULL','332','val_332' -'NULL','NULL','333','val_333' -'NULL','NULL','333','val_333' -'NULL','NULL','335','val_335' -'NULL','NULL','336','val_336' -'NULL','NULL','338','val_338' -'NULL','NULL','339','val_339' -'NULL','NULL','341','val_341' -'NULL','NULL','342','val_342' -'NULL','NULL','342','val_342' -'NULL','NULL','344','val_344' -'NULL','NULL','344','val_344' -'NULL','NULL','345','val_345' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','348','val_348' -'NULL','NULL','351','val_351' -'NULL','NULL','353','val_353' -'NULL','NULL','353','val_353' -'NULL','NULL','356','val_356' -'NULL','NULL','360','val_360' -'NULL','NULL','362','val_362' -'NULL','NULL','364','val_364' -'NULL','NULL','365','val_365' -'NULL','NULL','366','val_366' -'NULL','NULL','367','val_367' -'NULL','NULL','367','val_367' -'NULL','NULL','368','val_368' -'NULL','NULL','369','val_369' -'NULL','NULL','369','val_369' -'NULL','NULL','369','val_369' -'NULL','NULL','373','val_373' -'NULL','NULL','374','val_374' -'NULL','NULL','375','val_375' -'NULL','NULL','377','val_377' -'NULL','NULL','378','val_378' -'NULL','NULL','379','val_379' -'NULL','NULL','382','val_382' -'NULL','NULL','382','val_382' -'NULL','NULL','384','val_384' -'NULL','NULL','384','val_384' -'NULL','NULL','384','val_384' -'NULL','NULL','386','val_386' -'NULL','NULL','389','val_389' -'NULL','NULL','392','val_392' -'NULL','NULL','393','val_393' -'NULL','NULL','394','val_394' -'NULL','NULL','395','val_395' -'NULL','NULL','395','val_395' -'NULL','NULL','396','val_396' -'NULL','NULL','396','val_396' -'NULL','NULL','396','val_396' -'NULL','NULL','397','val_397' -'NULL','NULL','397','val_397' -'NULL','NULL','399','val_399' -'NULL','NULL','399','val_399' -'NULL','NULL','400','val_400' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','401','val_401' -'NULL','NULL','402','val_402' -'NULL','NULL','403','val_403' -'NULL','NULL','403','val_403' -'NULL','NULL','403','val_403' -'NULL','NULL','404','val_404' -'NULL','NULL','404','val_404' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','406','val_406' -'NULL','NULL','407','val_407' -'NULL','NULL','409','val_409' -'NULL','NULL','409','val_409' -'NULL','NULL','409','val_409' -'NULL','NULL','411','val_411' -'NULL','NULL','413','val_413' -'NULL','NULL','413','val_413' -'NULL','NULL','414','val_414' -'NULL','NULL','414','val_414' -'NULL','NULL','417','val_417' -'NULL','NULL','417','val_417' -'NULL','NULL','417','val_417' -'NULL','NULL','418','val_418' -'NULL','NULL','419','val_419' -'NULL','NULL','421','val_421' -'NULL','NULL','424','val_424' -'NULL','NULL','424','val_424' -'NULL','NULL','427','val_427' -'NULL','NULL','429','val_429' -'NULL','NULL','429','val_429' -'NULL','NULL','430','val_430' -'NULL','NULL','430','val_430' -'NULL','NULL','430','val_430' -'NULL','NULL','431','val_431' -'NULL','NULL','431','val_431' -'NULL','NULL','431','val_431' -'NULL','NULL','432','val_432' -'NULL','NULL','435','val_435' -'NULL','NULL','436','val_436' -'NULL','NULL','437','val_437' -'NULL','NULL','438','val_438' -'NULL','NULL','438','val_438' -'NULL','NULL','438','val_438' -'NULL','NULL','439','val_439' -'NULL','NULL','439','val_439' -'NULL','NULL','443','val_443' -'NULL','NULL','444','val_444' -'NULL','NULL','446','val_446' -'NULL','NULL','448','val_448' -'NULL','NULL','449','val_449' -'NULL','NULL','452','val_452' -'NULL','NULL','453','val_453' -'NULL','NULL','454','val_454' -'NULL','NULL','454','val_454' -'NULL','NULL','454','val_454' -'NULL','NULL','455','val_455' -'NULL','NULL','457','val_457' -'NULL','NULL','458','val_458' -'NULL','NULL','458','val_458' -'NULL','NULL','459','val_459' -'NULL','NULL','459','val_459' -'NULL','NULL','460','val_460' -'NULL','NULL','462','val_462' -'NULL','NULL','462','val_462' -'NULL','NULL','463','val_463' -'NULL','NULL','463','val_463' -'NULL','NULL','466','val_466' -'NULL','NULL','466','val_466' -'NULL','NULL','466','val_466' -'NULL','NULL','467','val_467' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','468','val_468' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','469','val_469' -'NULL','NULL','470','val_470' -'NULL','NULL','472','val_472' -'NULL','NULL','475','val_475' -'NULL','NULL','477','val_477' -'NULL','NULL','478','val_478' -'NULL','NULL','478','val_478' -'NULL','NULL','479','val_479' -'NULL','NULL','480','val_480' -'NULL','NULL','480','val_480' -'NULL','NULL','480','val_480' -'NULL','NULL','481','val_481' -'NULL','NULL','482','val_482' -'NULL','NULL','483','val_483' -'NULL','NULL','484','val_484' -'NULL','NULL','485','val_485' -'NULL','NULL','487','val_487' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','489','val_489' -'NULL','NULL','490','val_490' -'NULL','NULL','491','val_491' -'NULL','NULL','492','val_492' -'NULL','NULL','492','val_492' -'NULL','NULL','493','val_493' -'NULL','NULL','494','val_494' -'NULL','NULL','495','val_495' -'NULL','NULL','496','val_496' -'NULL','NULL','497','val_497' -'NULL','NULL','498','val_498' -'NULL','NULL','498','val_498' -'NULL','NULL','498','val_498' -500 rows selected ->>> ->>> insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key; -INFO : Compiling commandqueryId=(!!{queryId}!!): insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:a.key, type:int, comment:null), FieldSchema(name:a.value, type:string, comment:null), FieldSchema(name:b.key, type:int, comment:null), FieldSchema(name:b.value, type:string, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: query: insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : PREHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : PREHOOK: Output: smb_mapjoin_7@normal_join_results -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks not specified. Estimated from input data size: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:4 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : Starting task [Stage-0:MOVE] in serial mode -INFO : Loading data to table smb_mapjoin_7.normal_join_results from file:/!!ELIDED!! -INFO : Starting task [Stage-2:STATS] in serial mode -INFO : POSTHOOK: query: insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_1 -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_bucket4_2 -INFO : POSTHOOK: Output: smb_mapjoin_7@normal_join_results -INFO : POSTHOOK: Lineage: normal_join_results.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: normal_join_results.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] -INFO : POSTHOOK: Lineage: normal_join_results.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] -INFO : POSTHOOK: Lineage: normal_join_results.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key -No rows affected ->>> ->>> select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results; -INFO : Compiling commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:k1, type:bigint, comment:null), FieldSchema(name:k2, type:bigint, comment:null), FieldSchema(name:v1, type:bigint, comment:null), FieldSchema(name:v2, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results -INFO : PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@normal_join_results -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@normal_join_results -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results -'k1','k2','v1','v2' -'0','130091','0','36210398070' -1 row selected ->>> select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results; -INFO : Compiling commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:k1, type:bigint, comment:null), FieldSchema(name:k2, type:bigint, comment:null), FieldSchema(name:v1, type:bigint, comment:null), FieldSchema(name:v2, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results -INFO : PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_join_results -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_join_results -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results -'k1','k2','v1','v2' -'0','130091','0','36210398070' -1 row selected ->>> select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable; -INFO : Compiling commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable -INFO : Semantic Analysis Completed -INFO : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:k1, type:bigint, comment:null), FieldSchema(name:k2, type:bigint, comment:null), FieldSchema(name:v1, type:bigint, comment:null), FieldSchema(name:v2, type:bigint, comment:null)], properties:null) -INFO : Completed compiling commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : Executing commandqueryId=(!!{queryId}!!): select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable -INFO : PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable -INFO : PREHOOK: type: QUERY -INFO : PREHOOK: Input: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : PREHOOK: Output: file:/!!ELIDED!! -WARN : Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. -INFO : Query ID = !!{queryId}!! -INFO : Total jobs = 1 -INFO : Launching Job 1 out of 1 -INFO : Starting task [Stage-1:MAPRED] in serial mode -INFO : Number of reduce tasks determined at compile time: 1 -INFO : In order to change the average load for a reducer (in bytes): -INFO : set hive.exec.reducers.bytes.per.reducer= -INFO : In order to limit the maximum number of reducers: -INFO : set hive.exec.reducers.max= -INFO : In order to set a constant number of reducers: -INFO : set mapreduce.job.reduces= -DEBUG : Configuring job !!{jobId}}!! with file:/!!ELIDED!! as the submit dir -DEBUG : adding the following namenodes' delegation tokens:[file:///] -DEBUG : Creating splits at file:/!!ELIDED!! -INFO : number of splits:1 -INFO : Submitting tokens for job: !!{jobId}}!! -INFO : The url to track the job: http://localhost:8080/ -INFO : Job running in-process (local Hadoop) -INFO : Ended Job = !!{jobId}!! -INFO : POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable -INFO : POSTHOOK: type: QUERY -INFO : POSTHOOK: Input: smb_mapjoin_7@smb_join_results_empty_bigtable -INFO : POSTHOOK: Output: file:/!!ELIDED!! -INFO : MapReduce Jobs Launched: -INFO : Stage-Stage-1: HDFS Read: 0 HDFS Write: 0 SUCCESS -INFO : Total MapReduce CPU Time Spent: 0 msec -INFO : Completed executing commandqueryId=(!!{queryId}!!); Time taken: !!ELIDED!! seconds -INFO : OK -DEBUG : Shutting down query select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable -'k1','k2','v1','v2' -'0','130091','0','36210398070' -1 row selected ->>> ->>> ->>> ->>> ->>> ->>> ->>> !record +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket4_1 +POSTHOOK: Input: default@smb_bucket4_2 +POSTHOOK: Output: default@smb_join_results +POSTHOOK: Lineage: smb_join_results.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_join_results.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: smb_join_results.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from smb_join_results order by k1, v1, k2, v2 +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_join_results +#### A masked pattern was here #### +POSTHOOK: query: select * from smb_join_results order by k1, v1, k2, v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_join_results +#### A masked pattern was here #### +NULL NULL 0 val_0 +NULL NULL 0 val_0 +NULL NULL 0 val_0 +NULL NULL 2 val_2 +NULL NULL 4 val_4 +NULL NULL 5 val_5 +NULL NULL 5 val_5 +NULL NULL 5 val_5 +NULL NULL 8 val_8 +NULL NULL 9 val_9 +NULL NULL 10 val_10 +NULL NULL 11 val_11 +NULL NULL 12 val_12 +NULL NULL 12 val_12 +NULL NULL 15 val_15 +NULL NULL 15 val_15 +NULL NULL 17 val_17 +NULL NULL 18 val_18 +NULL NULL 18 val_18 +NULL NULL 19 val_19 +NULL NULL 20 val_20 +NULL NULL 24 val_24 +NULL NULL 24 val_24 +NULL NULL 26 val_26 +NULL NULL 26 val_26 +NULL NULL 27 val_27 +NULL NULL 28 val_28 +NULL NULL 30 val_30 +NULL NULL 33 val_33 +NULL NULL 34 val_34 +NULL NULL 35 val_35 +NULL NULL 35 val_35 +NULL NULL 35 val_35 +NULL NULL 37 val_37 +NULL NULL 37 val_37 +NULL NULL 41 val_41 +NULL NULL 42 val_42 +NULL NULL 42 val_42 +NULL NULL 43 val_43 +NULL NULL 44 val_44 +NULL NULL 47 val_47 +NULL NULL 51 val_51 +NULL NULL 51 val_51 +NULL NULL 53 val_53 +NULL NULL 54 val_54 +NULL NULL 57 val_57 +NULL NULL 58 val_58 +NULL NULL 58 val_58 +NULL NULL 64 val_64 +NULL NULL 65 val_65 +NULL NULL 66 val_66 +NULL NULL 67 val_67 +NULL NULL 67 val_67 +NULL NULL 69 val_69 +NULL NULL 70 val_70 +NULL NULL 70 val_70 +NULL NULL 70 val_70 +NULL NULL 72 val_72 +NULL NULL 72 val_72 +NULL NULL 74 val_74 +NULL NULL 76 val_76 +NULL NULL 76 val_76 +NULL NULL 77 val_77 +NULL NULL 78 val_78 +NULL NULL 80 val_80 +NULL NULL 82 val_82 +NULL NULL 83 val_83 +NULL NULL 83 val_83 +NULL NULL 84 val_84 +NULL NULL 84 val_84 +NULL NULL 85 val_85 +NULL NULL 86 val_86 +NULL NULL 87 val_87 +NULL NULL 90 val_90 +NULL NULL 90 val_90 +NULL NULL 90 val_90 +NULL NULL 92 val_92 +NULL NULL 95 val_95 +NULL NULL 95 val_95 +NULL NULL 96 val_96 +NULL NULL 97 val_97 +NULL NULL 97 val_97 +NULL NULL 98 val_98 +NULL NULL 98 val_98 +NULL NULL 100 val_100 +NULL NULL 100 val_100 +NULL NULL 103 val_103 +NULL NULL 103 val_103 +NULL NULL 104 val_104 +NULL NULL 104 val_104 +NULL NULL 105 val_105 +NULL NULL 111 val_111 +NULL NULL 113 val_113 +NULL NULL 113 val_113 +NULL NULL 114 val_114 +NULL NULL 116 val_116 +NULL NULL 118 val_118 +NULL NULL 118 val_118 +NULL NULL 119 val_119 +NULL NULL 119 val_119 +NULL NULL 119 val_119 +NULL NULL 120 val_120 +NULL NULL 120 val_120 +NULL NULL 125 val_125 +NULL NULL 125 val_125 +NULL NULL 126 val_126 +NULL NULL 128 val_128 +NULL NULL 128 val_128 +NULL NULL 128 val_128 +NULL NULL 129 val_129 +NULL NULL 129 val_129 +NULL NULL 131 val_131 +NULL NULL 133 val_133 +NULL NULL 134 val_134 +NULL NULL 134 val_134 +NULL NULL 136 val_136 +NULL NULL 137 val_137 +NULL NULL 137 val_137 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 138 val_138 +NULL NULL 143 val_143 +NULL NULL 145 val_145 +NULL NULL 146 val_146 +NULL NULL 146 val_146 +NULL NULL 149 val_149 +NULL NULL 149 val_149 +NULL NULL 150 val_150 +NULL NULL 152 val_152 +NULL NULL 152 val_152 +NULL NULL 153 val_153 +NULL NULL 155 val_155 +NULL NULL 156 val_156 +NULL NULL 157 val_157 +NULL NULL 158 val_158 +NULL NULL 160 val_160 +NULL NULL 162 val_162 +NULL NULL 163 val_163 +NULL NULL 164 val_164 +NULL NULL 164 val_164 +NULL NULL 165 val_165 +NULL NULL 165 val_165 +NULL NULL 166 val_166 +NULL NULL 167 val_167 +NULL NULL 167 val_167 +NULL NULL 167 val_167 +NULL NULL 168 val_168 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 169 val_169 +NULL NULL 170 val_170 +NULL NULL 172 val_172 +NULL NULL 172 val_172 +NULL NULL 174 val_174 +NULL NULL 174 val_174 +NULL NULL 175 val_175 +NULL NULL 175 val_175 +NULL NULL 176 val_176 +NULL NULL 176 val_176 +NULL NULL 177 val_177 +NULL NULL 178 val_178 +NULL NULL 179 val_179 +NULL NULL 179 val_179 +NULL NULL 180 val_180 +NULL NULL 181 val_181 +NULL NULL 183 val_183 +NULL NULL 186 val_186 +NULL NULL 187 val_187 +NULL NULL 187 val_187 +NULL NULL 187 val_187 +NULL NULL 189 val_189 +NULL NULL 190 val_190 +NULL NULL 191 val_191 +NULL NULL 191 val_191 +NULL NULL 192 val_192 +NULL NULL 193 val_193 +NULL NULL 193 val_193 +NULL NULL 193 val_193 +NULL NULL 194 val_194 +NULL NULL 195 val_195 +NULL NULL 195 val_195 +NULL NULL 196 val_196 +NULL NULL 197 val_197 +NULL NULL 197 val_197 +NULL NULL 199 val_199 +NULL NULL 199 val_199 +NULL NULL 199 val_199 +NULL NULL 200 val_200 +NULL NULL 200 val_200 +NULL NULL 201 val_201 +NULL NULL 202 val_202 +NULL NULL 203 val_203 +NULL NULL 203 val_203 +NULL NULL 205 val_205 +NULL NULL 205 val_205 +NULL NULL 207 val_207 +NULL NULL 207 val_207 +NULL NULL 208 val_208 +NULL NULL 208 val_208 +NULL NULL 208 val_208 +NULL NULL 209 val_209 +NULL NULL 209 val_209 +NULL NULL 213 val_213 +NULL NULL 213 val_213 +NULL NULL 214 val_214 +NULL NULL 216 val_216 +NULL NULL 216 val_216 +NULL NULL 217 val_217 +NULL NULL 217 val_217 +NULL NULL 218 val_218 +NULL NULL 219 val_219 +NULL NULL 219 val_219 +NULL NULL 221 val_221 +NULL NULL 221 val_221 +NULL NULL 222 val_222 +NULL NULL 223 val_223 +NULL NULL 223 val_223 +NULL NULL 224 val_224 +NULL NULL 224 val_224 +NULL NULL 226 val_226 +NULL NULL 228 val_228 +NULL NULL 229 val_229 +NULL NULL 229 val_229 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 230 val_230 +NULL NULL 233 val_233 +NULL NULL 233 val_233 +NULL NULL 235 val_235 +NULL NULL 237 val_237 +NULL NULL 237 val_237 +NULL NULL 238 val_238 +NULL NULL 238 val_238 +NULL NULL 239 val_239 +NULL NULL 239 val_239 +NULL NULL 241 val_241 +NULL NULL 242 val_242 +NULL NULL 242 val_242 +NULL NULL 244 val_244 +NULL NULL 247 val_247 +NULL NULL 248 val_248 +NULL NULL 249 val_249 +NULL NULL 252 val_252 +NULL NULL 255 val_255 +NULL NULL 255 val_255 +NULL NULL 256 val_256 +NULL NULL 256 val_256 +NULL NULL 257 val_257 +NULL NULL 258 val_258 +NULL NULL 260 val_260 +NULL NULL 262 val_262 +NULL NULL 263 val_263 +NULL NULL 265 val_265 +NULL NULL 265 val_265 +NULL NULL 266 val_266 +NULL NULL 272 val_272 +NULL NULL 272 val_272 +NULL NULL 273 val_273 +NULL NULL 273 val_273 +NULL NULL 273 val_273 +NULL NULL 274 val_274 +NULL NULL 275 val_275 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 277 val_277 +NULL NULL 278 val_278 +NULL NULL 278 val_278 +NULL NULL 280 val_280 +NULL NULL 280 val_280 +NULL NULL 281 val_281 +NULL NULL 281 val_281 +NULL NULL 282 val_282 +NULL NULL 282 val_282 +NULL NULL 283 val_283 +NULL NULL 284 val_284 +NULL NULL 285 val_285 +NULL NULL 286 val_286 +NULL NULL 287 val_287 +NULL NULL 288 val_288 +NULL NULL 288 val_288 +NULL NULL 289 val_289 +NULL NULL 291 val_291 +NULL NULL 292 val_292 +NULL NULL 296 val_296 +NULL NULL 298 val_298 +NULL NULL 298 val_298 +NULL NULL 298 val_298 +NULL NULL 302 val_302 +NULL NULL 305 val_305 +NULL NULL 306 val_306 +NULL NULL 307 val_307 +NULL NULL 307 val_307 +NULL NULL 308 val_308 +NULL NULL 309 val_309 +NULL NULL 309 val_309 +NULL NULL 310 val_310 +NULL NULL 311 val_311 +NULL NULL 311 val_311 +NULL NULL 311 val_311 +NULL NULL 315 val_315 +NULL NULL 316 val_316 +NULL NULL 316 val_316 +NULL NULL 316 val_316 +NULL NULL 317 val_317 +NULL NULL 317 val_317 +NULL NULL 318 val_318 +NULL NULL 318 val_318 +NULL NULL 318 val_318 +NULL NULL 321 val_321 +NULL NULL 321 val_321 +NULL NULL 322 val_322 +NULL NULL 322 val_322 +NULL NULL 323 val_323 +NULL NULL 325 val_325 +NULL NULL 325 val_325 +NULL NULL 327 val_327 +NULL NULL 327 val_327 +NULL NULL 327 val_327 +NULL NULL 331 val_331 +NULL NULL 331 val_331 +NULL NULL 332 val_332 +NULL NULL 333 val_333 +NULL NULL 333 val_333 +NULL NULL 335 val_335 +NULL NULL 336 val_336 +NULL NULL 338 val_338 +NULL NULL 339 val_339 +NULL NULL 341 val_341 +NULL NULL 342 val_342 +NULL NULL 342 val_342 +NULL NULL 344 val_344 +NULL NULL 344 val_344 +NULL NULL 345 val_345 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 348 val_348 +NULL NULL 351 val_351 +NULL NULL 353 val_353 +NULL NULL 353 val_353 +NULL NULL 356 val_356 +NULL NULL 360 val_360 +NULL NULL 362 val_362 +NULL NULL 364 val_364 +NULL NULL 365 val_365 +NULL NULL 366 val_366 +NULL NULL 367 val_367 +NULL NULL 367 val_367 +NULL NULL 368 val_368 +NULL NULL 369 val_369 +NULL NULL 369 val_369 +NULL NULL 369 val_369 +NULL NULL 373 val_373 +NULL NULL 374 val_374 +NULL NULL 375 val_375 +NULL NULL 377 val_377 +NULL NULL 378 val_378 +NULL NULL 379 val_379 +NULL NULL 382 val_382 +NULL NULL 382 val_382 +NULL NULL 384 val_384 +NULL NULL 384 val_384 +NULL NULL 384 val_384 +NULL NULL 386 val_386 +NULL NULL 389 val_389 +NULL NULL 392 val_392 +NULL NULL 393 val_393 +NULL NULL 394 val_394 +NULL NULL 395 val_395 +NULL NULL 395 val_395 +NULL NULL 396 val_396 +NULL NULL 396 val_396 +NULL NULL 396 val_396 +NULL NULL 397 val_397 +NULL NULL 397 val_397 +NULL NULL 399 val_399 +NULL NULL 399 val_399 +NULL NULL 400 val_400 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 401 val_401 +NULL NULL 402 val_402 +NULL NULL 403 val_403 +NULL NULL 403 val_403 +NULL NULL 403 val_403 +NULL NULL 404 val_404 +NULL NULL 404 val_404 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 406 val_406 +NULL NULL 407 val_407 +NULL NULL 409 val_409 +NULL NULL 409 val_409 +NULL NULL 409 val_409 +NULL NULL 411 val_411 +NULL NULL 413 val_413 +NULL NULL 413 val_413 +NULL NULL 414 val_414 +NULL NULL 414 val_414 +NULL NULL 417 val_417 +NULL NULL 417 val_417 +NULL NULL 417 val_417 +NULL NULL 418 val_418 +NULL NULL 419 val_419 +NULL NULL 421 val_421 +NULL NULL 424 val_424 +NULL NULL 424 val_424 +NULL NULL 427 val_427 +NULL NULL 429 val_429 +NULL NULL 429 val_429 +NULL NULL 430 val_430 +NULL NULL 430 val_430 +NULL NULL 430 val_430 +NULL NULL 431 val_431 +NULL NULL 431 val_431 +NULL NULL 431 val_431 +NULL NULL 432 val_432 +NULL NULL 435 val_435 +NULL NULL 436 val_436 +NULL NULL 437 val_437 +NULL NULL 438 val_438 +NULL NULL 438 val_438 +NULL NULL 438 val_438 +NULL NULL 439 val_439 +NULL NULL 439 val_439 +NULL NULL 443 val_443 +NULL NULL 444 val_444 +NULL NULL 446 val_446 +NULL NULL 448 val_448 +NULL NULL 449 val_449 +NULL NULL 452 val_452 +NULL NULL 453 val_453 +NULL NULL 454 val_454 +NULL NULL 454 val_454 +NULL NULL 454 val_454 +NULL NULL 455 val_455 +NULL NULL 457 val_457 +NULL NULL 458 val_458 +NULL NULL 458 val_458 +NULL NULL 459 val_459 +NULL NULL 459 val_459 +NULL NULL 460 val_460 +NULL NULL 462 val_462 +NULL NULL 462 val_462 +NULL NULL 463 val_463 +NULL NULL 463 val_463 +NULL NULL 466 val_466 +NULL NULL 466 val_466 +NULL NULL 466 val_466 +NULL NULL 467 val_467 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 468 val_468 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 469 val_469 +NULL NULL 470 val_470 +NULL NULL 472 val_472 +NULL NULL 475 val_475 +NULL NULL 477 val_477 +NULL NULL 478 val_478 +NULL NULL 478 val_478 +NULL NULL 479 val_479 +NULL NULL 480 val_480 +NULL NULL 480 val_480 +NULL NULL 480 val_480 +NULL NULL 481 val_481 +NULL NULL 482 val_482 +NULL NULL 483 val_483 +NULL NULL 484 val_484 +NULL NULL 485 val_485 +NULL NULL 487 val_487 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 489 val_489 +NULL NULL 490 val_490 +NULL NULL 491 val_491 +NULL NULL 492 val_492 +NULL NULL 492 val_492 +NULL NULL 493 val_493 +NULL NULL 494 val_494 +NULL NULL 495 val_495 +NULL NULL 496 val_496 +NULL NULL 497 val_497 +NULL NULL 498 val_498 +NULL NULL 498 val_498 +NULL NULL 498 val_498 +PREHOOK: query: insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket4_1 +PREHOOK: Input: default@smb_bucket4_2 +PREHOOK: Output: default@normal_join_results +POSTHOOK: query: insert overwrite table normal_join_results select * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket4_1 +POSTHOOK: Input: default@smb_bucket4_2 +POSTHOOK: Output: default@normal_join_results +POSTHOOK: Lineage: normal_join_results.k1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: normal_join_results.k2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: normal_join_results.v1 SIMPLE [(smb_bucket4_1)a.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: normal_join_results.v2 SIMPLE [(smb_bucket4_2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results +PREHOOK: type: QUERY +PREHOOK: Input: default@normal_join_results +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from normal_join_results +POSTHOOK: type: QUERY +POSTHOOK: Input: default@normal_join_results +#### A masked pattern was here #### +0 130091 0 36210398070 +PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_join_results +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_join_results +#### A masked pattern was here #### +0 130091 0 36210398070 +PREHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_join_results_empty_bigtable +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(k1)) as k1, sum(hash(k2)) as k2, sum(hash(v1)) as v1, sum(hash(v2)) as v2 from smb_join_results_empty_bigtable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_join_results_empty_bigtable +#### A masked pattern was here #### +0 130091 0 36210398070 diff --git service/src/java/org/apache/hive/service/cli/operation/OperationManager.java service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index f62ee4e..ac64ab2 100644 --- service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.log.LogDivertAppender; +import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest; import org.apache.hadoop.hive.ql.session.OperationLog; import org.apache.hive.service.AbstractService; import org.apache.hive.service.cli.FetchOrientation; @@ -71,6 +72,7 @@ public OperationManager() { @Override public synchronized void init(HiveConf hiveConf) { LogDivertAppender.registerRoutingAppender(hiveConf); + LogDivertAppenderForTest.registerRoutingAppenderIfInTest(hiveConf); if (hiveConf.isWebUiQueryInfoCacheEnabled()) { historicSqlOperations = new SQLOperationDisplayCache( diff --git service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index 7df4563..f5d7ec0 100644 --- service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -777,7 +777,8 @@ public void close() throws HiveSQLException { } private void cleanupSessionLogDir() { - if (isOperationLogEnabled) { + // In case of test, if we might not want to remove the log directory + if (isOperationLogEnabled && sessionConf.getBoolVar(ConfVars.HIVE_IN_TEST_REMOVE_LOGS)) { try { FileUtils.forceDelete(sessionLogDir); LOG.info("Operation log session directory is deleted: "