Index: . =================================================================== --- . (revision 1653459) +++ . (working copy) Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /hive/trunk:r1651028-1653743 Index: beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java =================================================================== --- beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java (revision 1653459) +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java (working copy) @@ -17,6 +17,7 @@ */ package org.apache.hive.beeline; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; @@ -97,55 +98,65 @@ } static final String DEFAUTL_DELIMITER = ";"; - /*** + + /** * Find the type of given command + * * @param dbCommand * @return */ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; - /** Parse the DB specific nesting format and extract the inner script name if any + /** + * Parse the DB specific nesting format and extract the inner script name if any + * * @param dbCommand command from parent script * @return * @throws IllegalFormatException */ public String getScriptName(String dbCommand) throws IllegalArgumentException; - /*** + /** * Find if the given command is a nested script execution + * * @param dbCommand * @return */ public boolean isNestedScript(String dbCommand); - /*** + /** * Find if the given command should not be passed to DB + * * @param dbCommand * @return */ public boolean isNonExecCommand(String dbCommand); - /*** + /** * Get the SQL statement delimiter + * * @return */ public String getDelimiter(); - /*** + /** * Clear any client specific tags + * * @return */ public String cleanseCommand(String dbCommand); - /*** + /** * Does the DB required table/column names quoted + * * @return */ public boolean needsQuotedIdentifier(); - /*** + /** * Flatten the nested upgrade script into a buffer - * @param scriptDir upgrade script directory + * + * @param scriptDir upgrade script directory * @param scriptFile upgrade script file * @return string of sql commands */ @@ -258,6 +269,8 @@ private void setDbOpts(String dbOpts) { if (dbOpts != null) { this.dbOpts = Lists.newArrayList(dbOpts.split(",")); + } else { + this.dbOpts = Lists.newArrayList(); } } @@ -369,6 +382,10 @@ // Postgres specific parser public static class PostgresCommandParser extends AbstractCommandParser { private static String POSTGRES_NESTING_TOKEN = "\\i"; + @VisibleForTesting + public static String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings"; + @VisibleForTesting + public static String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81"; public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, HiveConf hiveConf) { @@ -394,6 +411,19 @@ public boolean needsQuotedIdentifier() { return true; } + + @Override + public boolean isNonExecCommand(String dbCommand) { + // Skip "standard_conforming_strings" command which is read-only in older + // Postgres versions like 8.1 + // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html + if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) { + if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) { + return true; + } + } + return super.isNonExecCommand(dbCommand); + } } //Oracle specific parser Index: beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java =================================================================== --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java (revision 1653459) +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java (working copy) @@ -178,7 +178,8 @@ getConnectionToMetastore(false)); // verify that the new version is added to schema if (!MetaStoreSchemaInfo.getHiveSchemaVersion().equalsIgnoreCase(newSchemaVersion)) { - throw new HiveMetaException("Found unexpected schema version " + newSchemaVersion); + throw new HiveMetaException("Expected schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion() + + ", found version " + newSchemaVersion); } } Index: cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java =================================================================== --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (revision 1653459) +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (working copy) @@ -40,6 +40,7 @@ import jline.console.ConsoleReader; import jline.console.completer.Completer; import jline.console.history.FileHistory; +import jline.console.history.PersistentHistory; import jline.console.completer.StringsCompleter; import jline.console.completer.ArgumentCompleter; import jline.console.completer.ArgumentCompleter.ArgumentDelimiter; @@ -721,10 +722,12 @@ String line; final String HISTORYFILE = ".hivehistory"; String historyDirectory = System.getProperty("user.home"); + PersistentHistory history = null; try { if ((new File(historyDirectory)).exists()) { String historyFile = historyDirectory + File.separator + HISTORYFILE; - reader.setHistory(new FileHistory(new File(historyFile))); + history = new FileHistory(new File(historyFile)); + reader.setHistory(history); } else { System.err.println("WARNING: Directory for Hive history file: " + historyDirectory + " does not exist. History will not be available during this session."); @@ -759,6 +762,10 @@ continue; } } + + if (history != null) { + history.flush(); + } return ret; } Index: common/src/java/org/apache/hadoop/hive/common/FileUtils.java =================================================================== --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java (revision 1653459) +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java (working copy) @@ -53,17 +53,19 @@ public final class FileUtils { private static final Log LOG = LogFactory.getLog(FileUtils.class.getName()); - /** - * Accept all paths. - */ - private static class AcceptAllPathFilter implements PathFilter { - @Override - public boolean accept(Path path) { - return true; + public static final PathFilter HIDDEN_FILES_PATH_FILTER = new PathFilter() { + public boolean accept(Path p) { + String name = p.getName(); + return !name.startsWith("_") && !name.startsWith("."); } - } + }; - private static final PathFilter allPathFilter = new AcceptAllPathFilter(); + public static final PathFilter STAGING_DIR_PATH_FILTER = new PathFilter() { + public boolean accept(Path p) { + String name = p.getName(); + return !name.startsWith("."); + } + }; /** * Variant of Path.makeQualified that qualifies the input path against the default file system @@ -319,14 +321,7 @@ List results) throws IOException { if (fileStatus.isDir()) { - for (FileStatus stat : fs.listStatus(fileStatus.getPath(), new PathFilter() { - - @Override - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - })) { + for (FileStatus stat : fs.listStatus(fileStatus.getPath(), HIDDEN_FILES_PATH_FILTER)) { listStatusRecursively(fs, stat, results); } } else { @@ -366,7 +361,6 @@ * check will be performed within a doAs() block to use the access privileges * of this user. In this case the user must be configured to impersonate other * users, otherwise this check will fail with error. - * @param groups List of groups for the user * @throws IOException * @throws AccessControlException * @throws InterruptedException @@ -547,10 +541,25 @@ boolean deleteSource, boolean overwrite, HiveConf conf) throws IOException { - boolean copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf); + + HadoopShims shims = ShimLoader.getHadoopShims(); + boolean copied; + + /* Run distcp if source file/dir is too big */ + if (srcFS.getUri().getScheme().equals("hdfs") && + srcFS.getFileStatus(src).getLen() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE)) { + LOG.info("Source is " + srcFS.getFileStatus(src).getLen() + " bytes. (MAX: " + conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE) + ")"); + LOG.info("Launch distributed copy (distcp) job."); + copied = shims.runDistCp(src, dst, conf); + if (copied && deleteSource) { + srcFS.delete(src, true); + } + } else { + copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf); + } + boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); if (copied && inheritPerms) { - HadoopShims shims = ShimLoader.getHadoopShims(); HdfsFileStatus fullFileStatus = shims.getFullFileStatus(conf, dstFS, dst); try { shims.setFullFileStatus(conf, fullFileStatus, dstFS, dst); @@ -571,7 +580,7 @@ * @throws IOException */ public static boolean trashFilesUnderDir(FileSystem fs, Path f, Configuration conf) throws FileNotFoundException, IOException { - FileStatus[] statuses = fs.listStatus(f, allPathFilter); + FileStatus[] statuses = fs.listStatus(f, HIDDEN_FILES_PATH_FILTER); boolean result = true; for (FileStatus status : statuses) { result = result & moveToTrash(fs, status.getPath(), conf); @@ -603,6 +612,25 @@ return result; } + /** + * Check if first path is a subdirectory of second path. + * Both paths must belong to the same filesystem. + * + * @param p1 first path + * @param p2 second path + * @param fs FileSystem, both paths must belong to the same filesystem + * @return + */ + public static boolean isSubDir(Path p1, Path p2, FileSystem fs) { + String path1 = fs.makeQualified(p1).toString(); + String path2 = fs.makeQualified(p2).toString(); + if (path1.startsWith(path2)) { + return true; + } + + return false; + } + public static boolean renameWithPerms(FileSystem fs, Path sourcePath, Path destPath, boolean inheritPerms, Configuration conf) throws IOException { Index: common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java =================================================================== --- common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java (revision 1653459) +++ common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java (working copy) @@ -70,7 +70,7 @@ sb.append(Path.SEPARATOR).append("*"); } Path pathPattern = new Path(path, sb.toString()); - return fs.globStatus(pathPattern); + return fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER); } } Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1653459) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -215,6 +215,10 @@ PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo", "Query plan format serialization between client and task nodes. \n" + "Two supported values are : kryo and javaXML. Kryo is default."), + STAGINGDIR("hive.exec.stagingdir", ".hive-staging", + "Directory name that will be created inside table locations in order to support HDFS encryption. " + + "This is replaces ${hive.exec.scratchdir} for query results with the exception of read-only tables. " + + "In all cases ${hive.exec.scratchdir} is still used for other temporary files, such as job plans."), SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive", "HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. " + "For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/ is created, " + @@ -267,6 +271,10 @@ "Comma-separated list of on-failure hooks to be invoked for each statement. \n" + "An on-failure hook is specified as the name of Java class which implements the \n" + "org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext interface."), + QUERYREDACTORHOOKS("hive.exec.query.redactor.hooks", "", + "Comma-separated list of hooks to be invoked for each query which can \n" + + "tranform the query before it's placed in the job.xml file. Must be a Java class which \n" + + "extends from the org.apache.hadoop.hive.ql.hooks.Redactor abstract class."), CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", "", "Comma-separated list of statistics publishers to be invoked on counters on each job. \n" + "A client stats publisher is specified as the name of a Java class which implements the \n" + @@ -699,13 +707,6 @@ HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true, "Whether Hive should use memory-optimized hash table for MapJoin. Only works on Tez,\n" + "because memory-optimized hashtable cannot be serialized."), - HIVEMAPJOINUSEOPTIMIZEDKEYS("hive.mapjoin.optimized.keys", true, - "Whether MapJoin hashtable should use optimized (size-wise), keys, allowing the table to take less\n" + - "memory. Depending on key, the memory savings for entire table can be 5-15% or so."), - HIVEMAPJOINLAZYHASHTABLE("hive.mapjoin.lazy.hashtable", true, - "Whether MapJoin hashtable should deserialize values on demand. Depending on how many values in\n" + - "the table the join will actually touch, it can save a lot of memory by not creating objects for\n" + - "rows that are not needed. If all rows are needed obviously there's no gain."), HIVEHASHTABLEWBSIZE("hive.mapjoin.optimized.hashtable.wbsize", 10 * 1024 * 1024, "Optimized hashtable (see hive.mapjoin.optimized.hashtable) uses a chain of buffers to\n" + "store data. This is one buffer size. HT may be slightly faster if this is larger, but for small\n" + @@ -749,6 +750,10 @@ "cardinality (4 in the example above), is more than this value, a new MR job is added under the\n" + "assumption that the original group by will reduce the data size."), + // Max filesize used to do a single copy (after that, distcp is used) + HIVE_EXEC_COPYFILE_MAXSIZE("hive.exec.copyfile.maxsize", 32L * 1024 * 1024 /*32M*/, + "Maximum file size (in Mb) that Hive uses to do single HDFS copies between directories." + + "Distributed copies (distcp) will be used instead for bigger files so that copies can be done faster."), // for hive udtf operator HIVEUDTFAUTOPROGRESS("hive.udtf.auto.progress", false, @@ -1323,13 +1328,20 @@ "The port of ZooKeeper servers to talk to.\n" + "If the list of Zookeeper servers specified in hive.zookeeper.quorum\n" + "does not contain port numbers, this value is used."), - HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000, - "ZooKeeper client's session timeout. The client is disconnected, and as a result, all locks released, \n" + + HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", "600000ms", + new TimeValidator(TimeUnit.MILLISECONDS), + "ZooKeeper client's session timeout (in milliseconds). The client is disconnected, and as a result, all locks released, \n" + "if a heartbeat is not sent in the timeout."), HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace", "hive_zookeeper_namespace", "The parent node under which all ZooKeeper nodes are created."), HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes", false, "Clean extra nodes at the end of the session."), + HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES("hive.zookeeper.connection.max.retries", 3, + "Max number of times to retry when connecting to the ZooKeeper server."), + HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME("hive.zookeeper.connection.basesleeptime", "1000ms", + new TimeValidator(TimeUnit.MILLISECONDS), + "Initial amount of time (in milliseconds) to wait between retries\n" + + "when connecting to the ZooKeeper server when using ExponentialBackoffRetry policy."), // Transactions HIVE_TXN_MANAGER("hive.txn.manager", @@ -1608,6 +1620,10 @@ "inheriting the permission of the warehouse or database directory."), HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true, "whether insert into external tables is allowed"), + HIVE_TEMPORARY_TABLE_STORAGE( + "hive.exec.temporary.table.storage", "default", new StringSet("memory", + "ssd", "default"), "Define the storage policy for temporary tables." + + "Choices between memory, ssd and default"), HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", "", "A comma separated list of hooks which implement HiveDriverRunHook. Will be run at the beginning " + Index: data/scripts/q_test_cleanup_for_encryption.sql =================================================================== --- data/scripts/q_test_cleanup_for_encryption.sql (revision 0) +++ data/scripts/q_test_cleanup_for_encryption.sql (working copy) @@ -0,0 +1 @@ +DROP TABLE IF EXISTS src; Index: data/scripts/q_test_init_for_encryption.sql =================================================================== --- data/scripts/q_test_init_for_encryption.sql (revision 0) +++ data/scripts/q_test_init_for_encryption.sql (working copy) @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS src; + +CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; Index: dev-support/jenkins-submit-build.sh =================================================================== --- dev-support/jenkins-submit-build.sh (revision 1653459) +++ dev-support/jenkins-submit-build.sh (working copy) @@ -30,12 +30,18 @@ curl -v -i "$url" exit 0 ;; - spark-mr2|spark2-mr2) + spark-mr2) test -n "$SPARK_URL" || fail "SPARK_URL must be specified" url="$SPARK_URL&ISSUE_NUM=$ISSUE_NUM" curl -v -i "$url" exit 0 ;; + encryption-mr2) + test -n "$ENCRYPTION_URL" || fail "ENCRYPTION_URL must be specified" + url="$ENCRYPTION_URL&ISSUE_NUM=$ISSUE_NUM" + curl -v -i "$url" + exit 0 + ;; *) echo "Unknown profile '$BUILD_PROFILE'" exit 1 Index: hbase-handler/pom.xml =================================================================== --- hbase-handler/pom.xml (revision 1653459) +++ hbase-handler/pom.xml (working copy) Property changes on: hbase-handler/pom.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /hive/trunk/hbase-handler/pom.xml:r1651028-1653696 Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java (revision 1653459) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java (working copy) @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.shims.ShimLoader; @@ -149,7 +150,7 @@ fs.mkdirs(columnFamilyPath); Path srcDir = outputdir; for (;;) { - FileStatus [] files = fs.listStatus(srcDir); + FileStatus [] files = fs.listStatus(srcDir, FileUtils.STAGING_DIR_PATH_FILTER); if ((files == null) || (files.length == 0)) { throw new IOException("No family directories found in " + srcDir); } @@ -161,7 +162,7 @@ break; } } - for (FileStatus regionFile : fs.listStatus(srcDir)) { + for (FileStatus regionFile : fs.listStatus(srcDir, FileUtils.STAGING_DIR_PATH_FILTER)) { fs.rename( regionFile.getPath(), new Path( Index: hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java =================================================================== --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java (revision 1653459) +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java (working copy) @@ -96,6 +96,19 @@ public static final String HCAT_DESIRED_PARTITION_NUM_SPLITS = "hcat.desired.partition.num.splits"; + /** + * hcat.append.limit allows a hcat user to specify a custom append limit. + * By default, while appending to an existing directory, hcat will attempt + * to avoid naming clashes and try to append _a_NNN where NNN is a number to + * the desired filename to avoid clashes. However, by default, it only tries + * for NNN from 0 to 999 before giving up. This can cause an issue for some + * tables with an extraordinarily large number of files. Ideally, this should + * be fixed by the user changing their usage pattern and doing some manner of + * compaction, but in the meanwhile, until they can, setting this parameter + * can be used to bump that limit. + */ + public static final String HCAT_APPEND_LIMIT = "hcat.append.limit"; + // IMPORTANT IMPORTANT IMPORTANT!!!!! //The keys used to store info into the job Configuration. //If any new keys are added, the HCatStorer needs to be updated. The HCatStorer @@ -132,8 +145,10 @@ public static final String HCAT_EVENT = "HCAT_EVENT"; public static final String HCAT_ADD_PARTITION_EVENT = "ADD_PARTITION"; public static final String HCAT_DROP_PARTITION_EVENT = "DROP_PARTITION"; + public static final String HCAT_ALTER_PARTITION_EVENT = "ALTER_PARTITION"; public static final String HCAT_PARTITION_DONE_EVENT = "PARTITION_DONE"; public static final String HCAT_CREATE_TABLE_EVENT = "CREATE_TABLE"; + public static final String HCAT_ALTER_TABLE_EVENT = "ALTER_TABLE"; public static final String HCAT_DROP_TABLE_EVENT = "DROP_TABLE"; public static final String HCAT_CREATE_DATABASE_EVENT = "CREATE_DATABASE"; public static final String HCAT_DROP_DATABASE_EVENT = "DROP_DATABASE"; Index: hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java =================================================================== --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (revision 1653459) +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (working copy) @@ -75,6 +75,8 @@ static final String DYNTEMP_DIR_NAME = "_DYN"; static final String SCRATCH_DIR_NAME = "_SCRATCH"; private static final String APPEND_SUFFIX = "_a_"; + private static final int APPEND_COUNTER_WARN_THRESHOLD = 1000; + private final int maxAppendAttempts; private static final Logger LOG = LoggerFactory.getLogger(FileOutputCommitterContainer.class); private final boolean dynamicPartitioningUsed; @@ -112,6 +114,8 @@ } else { customDynamicLocationUsed = false; } + + this.maxAppendAttempts = context.getConfiguration().getInt(HCatConstants.HCAT_APPEND_LIMIT, APPEND_COUNTER_WARN_THRESHOLD); } @Override @@ -646,19 +650,23 @@ filetype = ""; } - // Attempt to find COUNTER_MAX possible alternatives to a filename by + // Attempt to find maxAppendAttempts possible alternatives to a filename by // appending _a_N and seeing if that destination also clashes. If we're // still clashing after that, give up. - final int COUNTER_MAX = 1000; int counter = 1; - for (; fs.exists(itemDest) && counter < COUNTER_MAX ; counter++) { + for (; fs.exists(itemDest) && counter < maxAppendAttempts; counter++) { itemDest = new Path(dest, name + (APPEND_SUFFIX + counter) + filetype); } - if (counter == COUNTER_MAX){ + if (counter == maxAppendAttempts){ throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Could not find a unique destination path for move: file = " + file + " , src = " + src + ", dest = " + dest); + } else if (counter > APPEND_COUNTER_WARN_THRESHOLD) { + LOG.warn("Append job used filename clash counter [" + counter + +"] which is greater than warning limit [" + APPEND_COUNTER_WARN_THRESHOLD + +"]. Please compact this table so that performance is not impacted." + + " Please see HIVE-9381 for details."); } } @@ -696,7 +704,7 @@ // LOG.info("Searching for "+dynPathSpec); Path pathPattern = new Path(dynPathSpec); - FileStatus[] status = fs.globStatus(pathPattern); + FileStatus[] status = fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER); partitionsDiscoveredByPath = new LinkedHashMap>(); contextDiscoveredByPath = new LinkedHashMap(); Index: hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java =================================================================== --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java (revision 1653459) +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java (working copy) @@ -20,9 +20,7 @@ package org.apache.hive.hcatalog.mapreduce; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java (working copy) @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; @@ -140,7 +141,7 @@ * @throws MetaException */ public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { - /*Table before = tableEvent.getOldTable(); + Table before = tableEvent.getOldTable(); Table after = tableEvent.getNewTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT, @@ -149,8 +150,7 @@ event.setDbName(after.getDbName()); event.setTableName(after.getTableName()); enqueue(event); - }*/ - // TODO - once HIVE-9175 is committed + } } /** @@ -187,7 +187,16 @@ * @throws MetaException */ public void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException { - // TODO, MessageFactory doesn't support Alter Partition yet. + Partition before = partitionEvent.getOldPartition(); + Partition after = partitionEvent.getNewPartition(); + NotificationEvent event = new NotificationEvent(0, now(), + HCatConstants.HCAT_ALTER_PARTITION_EVENT, + msgFactory.buildAlterPartitionMessage(before, after).toString()); + if (event != null) { + event.setDbName(before.getDbName()); + event.setTableName(before.getTableName()); + enqueue(event); + } } /** Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java (working copy) @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.hive.hcatalog.messaging.MessageFactory; import org.slf4j.Logger; @@ -116,7 +117,7 @@ testAndCreateConnection(); } - private static String getTopicName(Table table, ListenerEvent partitionEvent) { + private static String getTopicName(Table table) { return table.getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME); } @@ -129,7 +130,7 @@ if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); List partitions = partitionEvent.getPartitions(); - String topicName = getTopicName(table, partitionEvent); + String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildAddPartitionMessage(table, partitions), topicName); } else { @@ -144,6 +145,17 @@ } } + @Override + public void onAlterPartition(AlterPartitionEvent ape) throws MetaException { + if (ape.getStatus()) { + Partition before = ape.getOldPartition(); + Partition after = ape.getNewPartition(); + + String topicName = getTopicName(ape.getTable()); + send(messageFactory.buildAlterPartitionMessage(before, after), topicName); + } + } + /** * Send dropped partition notifications. Subscribers can receive these notifications for a * particular table by listening on a topic named "dbName.tableName" with message selector @@ -165,7 +177,7 @@ sd.setParameters(new HashMap()); sd.getSerdeInfo().setParameters(new HashMap()); sd.getSkewedInfo().setSkewedColNames(new ArrayList()); - String topicName = getTopicName(partitionEvent.getTable(), partitionEvent); + String topicName = getTopicName(partitionEvent.getTable()); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partition), topicName); } else { @@ -241,6 +253,35 @@ } /** + * Send altered table notifications. Subscribers can receive these notifications for + * dropped tables by listening on topic "HCAT" with message selector string + * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_EVENT} = + * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_ALTER_TABLE_EVENT} + */ + @Override + public void onAlterTable(AlterTableEvent tableEvent) throws MetaException { + if (tableEvent.getStatus()) { + Table before = tableEvent.getOldTable(); + Table after = tableEvent.getNewTable(); + + // onCreateTable alters the table to add the topic name. Since this class is generating + // that alter, we don't want to notify on that alter. So take a quick look and see if + // that's what this this alter is, and if so swallow it. + if (after.getParameters() != null && + after.getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME) != null && + (before.getParameters() == null || + before.getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME) == null)) { + return; + } + // I think this is wrong, the alter table statement should come on the table topic not the + // DB topic - Alan. + String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + + after.getDbName().toLowerCase(); + send(messageFactory.buildAlterTableMessage(before, after), topicName); + } + } + + /** * Send dropped table notifications. Subscribers can receive these notifications for * dropped tables by listening on topic "HCAT" with message selector string * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_EVENT} = @@ -262,6 +303,8 @@ if (tableEvent.getStatus()) { Table table = tableEvent.getTable(); + // I think this is wrong, the drop table statement should come on the table topic not the + // DB topic - Alan. String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + table.getDbName().toLowerCase(); send(messageFactory.buildDropTableMessage(table), topicName); } @@ -435,14 +478,4 @@ // if(lpde.getStatus()) // send(lpde.getPartitionName(),lpde.getTable().getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME),HCatConstants.HCAT_PARTITION_DONE_EVENT); } - - @Override - public void onAlterPartition(AlterPartitionEvent ape) throws MetaException { - // no-op - } - - @Override - public void onAlterTable(AlterTableEvent ate) throws MetaException { - // no-op - } } Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java (revision 0) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java (working copy) @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.messaging; + +import java.util.List; + +/** + * HCat message sent when a table is Altered. + */ +public abstract class AlterPartitionMessage extends HCatEventMessage { + + protected AlterPartitionMessage() { + super(EventType.ALTER_PARTITION); + } + + public abstract String getTable(); + + public abstract List getValues(); + + @Override + public HCatEventMessage checkValid() { + if (getTable() == null) throw new IllegalStateException("Table name unset."); + if (getValues() == null) throw new IllegalStateException("Partition values unset"); + return super.checkValid(); + } +} + Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java (revision 0) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java (working copy) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.messaging; + +/** + * HCat message sent when a table is Altered. + */ +public abstract class AlterTableMessage extends HCatEventMessage { + + protected AlterTableMessage() { + super(EventType.ALTER_TABLE); + } + + public abstract String getTable(); + + @Override + public HCatEventMessage checkValid() { + if (getTable() == null) throw new IllegalStateException("Table name unset."); + return super.checkValid(); + } +} Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java (working copy) @@ -37,7 +37,9 @@ CREATE_TABLE(HCatConstants.HCAT_CREATE_TABLE_EVENT), DROP_TABLE(HCatConstants.HCAT_DROP_TABLE_EVENT), ADD_PARTITION(HCatConstants.HCAT_ADD_PARTITION_EVENT), - DROP_PARTITION(HCatConstants.HCAT_DROP_PARTITION_EVENT); + DROP_PARTITION(HCatConstants.HCAT_DROP_PARTITION_EVENT), + ALTER_TABLE(HCatConstants.HCAT_ALTER_TABLE_EVENT), + ALTER_PARTITION(HCatConstants.HCAT_ALTER_PARTITION_EVENT); private String typeString; Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java (working copy) @@ -36,10 +36,14 @@ return getDropDatabaseMessage(messageBody); case CREATE_TABLE: return getCreateTableMessage(messageBody); + case ALTER_TABLE: + return getAlterTableMessage(messageBody); case DROP_TABLE: return getDropTableMessage(messageBody); case ADD_PARTITION: return getAddPartitionMessage(messageBody); + case ALTER_PARTITION: + return getAlterPartitionMessage(messageBody); case DROP_PARTITION: return getDropPartitionMessage(messageBody); @@ -64,6 +68,13 @@ public abstract CreateTableMessage getCreateTableMessage(String messageBody); /** + * Method to de-serialize AlterTableMessge + * @param messageBody string message + * @return object message + */ + public abstract AlterTableMessage getAlterTableMessage(String messageBody); + + /** * Method to de-serialize DropTableMessage instance. */ public abstract DropTableMessage getDropTableMessage(String messageBody); @@ -74,6 +85,13 @@ public abstract AddPartitionMessage getAddPartitionMessage(String messageBody); /** + * Method to deserialize AlterPartitionMessage + * @param messageBody the message in serialized form + * @return message in object form + */ + public abstract AlterPartitionMessage getAlterPartitionMessage(String messageBody); + + /** * Method to de-serialize DropPartitionMessage instance. */ public abstract DropPartitionMessage getDropPartitionMessage(String messageBody); Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java (working copy) @@ -119,6 +119,17 @@ public abstract CreateTableMessage buildCreateTableMessage(Table table); /** + * Factory method for AlterTableMessage. Unlike most of these calls, this one can return null, + * which means no message should be sent. This is because there are many flavors of alter + * table (add column, add partition, etc.). Some are covered elsewhere (like add partition) + * and some are not yet supported. + * @param before The table before the alter + * @param after The table after the alter + * @return + */ + public abstract AlterTableMessage buildAlterTableMessage(Table before, Table after); + + /** * Factory method for DropTableMessage. * @param table The Table being dropped. * @return DropTableMessage instance. @@ -144,6 +155,15 @@ public abstract AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec); /** + * Factory method for building AlterPartitionMessage + * @param before The partition before it was altered + * @param after The partition after it was altered + * @return a new AlterPartitionMessage + */ + public abstract AlterPartitionMessage buildAlterPartitionMessage(Partition before, + Partition after); + + /** * Factory method for DropPartitionMessage. * @param table The Table from which the partition is dropped. * @param partition The Partition being dropped. Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java (revision 0) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java (working copy) @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.messaging.json; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * JSON alter table message + */ +public class JSONAlterPartitionMessage extends AlterPartitionMessage { + + @JsonProperty + String server, servicePrincipal, db, table; + + @JsonProperty + Long timestamp; + + @JsonProperty + List values; + + + public JSONAlterPartitionMessage(String server, + String servicePrincipal, + String db, + String table, + List values, + Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + this.values = values; + checkValid(); + } + + + @Override + public String getServer() { + return server; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getDB() { + return db; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getTable() { + return table; + } + + @Override + public List getValues() { + return values; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception e) { + throw new IllegalArgumentException("Could not serialize: ", e); + } + } +} Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java (revision 0) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java (working copy) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.messaging.json; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +/** + * JSON alter table message + */ +public class JSONAlterTableMessage extends AlterTableMessage { + + @JsonProperty + String server, servicePrincipal, db, table; + + @JsonProperty + Long timestamp; + + public JSONAlterTableMessage(String server, + String servicePrincipal, + String db, + String table, + Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + checkValid(); + } + + + @Override + public String getServer() { + return server; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getDB() { + return db; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getTable() { + return table; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception e) { + throw new IllegalArgumentException("Could not serialize: ", e); + } + } +} Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java (working copy) @@ -19,7 +19,10 @@ package org.apache.hive.hcatalog.messaging.json; +import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; @@ -71,6 +74,17 @@ } @Override + public AlterTableMessage getAlterTableMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONAlterTableMessage.class); + } + catch (Exception exception) { + throw new IllegalArgumentException("Could not construct appropriate alter table type.", + exception); + } + } + + @Override public DropTableMessage getDropTableMessage(String messageBody) { try { return mapper.readValue(messageBody, JSONDropTableMessage.class); @@ -91,6 +105,15 @@ } @Override + public AlterPartitionMessage getAlterPartitionMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONAlterPartitionMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct AlterPartitionMessage.", e); + } + } + + @Override public DropPartitionMessage getDropPartitionMessage(String messageBody) { try { return mapper.readValue(messageBody, JSONDropPartitionMessage.class); Index: hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java =================================================================== --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java (revision 1653459) +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java (working copy) @@ -24,13 +24,12 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; -// TODO, once HIVE-9175 is committed -// import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; @@ -85,14 +84,11 @@ table.getTableName(), now()); } - // TODO Once HIVE-9175 is committed - /* @Override public AlterTableMessage buildAlterTableMessage(Table before, Table after) { return new JSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), before.getTableName(), now()); } - */ @Override public DropTableMessage buildDropTableMessage(Table table) { @@ -111,10 +107,16 @@ @InterfaceStability.Evolving public AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionSpec), System.currentTimeMillis()/1000); + table.getTableName(), getPartitionKeyValues(table, partitionSpec), now()); } @Override + public AlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) { + return new JSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, + before.getDbName(), before.getTableName(), before.getValues(), now()); + } + + @Override public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), now()); Index: hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java =================================================================== --- hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java (revision 1653459) +++ hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java (working copy) @@ -46,6 +46,8 @@ import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; @@ -104,7 +106,9 @@ HCatConstants.HCAT_CREATE_DATABASE_EVENT, HCatConstants.HCAT_CREATE_TABLE_EVENT, HCatConstants.HCAT_ADD_PARTITION_EVENT, + HCatConstants.HCAT_ALTER_PARTITION_EVENT, HCatConstants.HCAT_DROP_PARTITION_EVENT, + HCatConstants.HCAT_ALTER_TABLE_EVENT, HCatConstants.HCAT_DROP_TABLE_EVENT, HCatConstants.HCAT_DROP_DATABASE_EVENT); Assert.assertEquals(expectedMessages, actualMessages); @@ -120,7 +124,9 @@ kvs.put("b", "2011"); client.markPartitionForEvent("mydb", "mytbl", kvs, PartitionEventType.LOAD_DONE); + driver.run("alter table mytbl partition (b='2011') set fileformat orc"); driver.run("alter table mytbl drop partition(b='2011')"); + driver.run("alter table mytbl add columns (c int comment 'this is an int', d decimal(3,2))"); driver.run("drop table mytbl"); driver.run("drop database mydb"); } @@ -170,6 +176,20 @@ Assert.assertEquals("mytbl", ((AddPartitionMessage) message2).getTable()); Assert.assertEquals(1, ((AddPartitionMessage) message2).getPartitions().size()); Assert.assertEquals("2011", ((AddPartitionMessage) message2).getPartitions().get(0).get("b")); + } else if (event.equals(HCatConstants.HCAT_ALTER_PARTITION_EVENT)) { + Assert.assertEquals("topic://hcat.mydb.mytbl", msg.getJMSDestination().toString()); + // for alter partition events + AlterPartitionMessage message = deserializer.getAlterPartitionMessage(messageBody); + Assert.assertEquals("mytbl", message.getTable()); + Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(1, message.getValues().size()); + Assert.assertEquals("2011", message.getValues().get(0)); + HCatEventMessage message2 = MessagingUtils.getMessage(msg); + Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterPartitionMessage); + Assert.assertEquals("mydb", message2.getDB()); + Assert.assertEquals("mytbl", ((AlterPartitionMessage) message2).getTable()); + Assert.assertEquals(1, ((AlterPartitionMessage) message2).getValues().size()); + Assert.assertEquals("2011", ((AlterPartitionMessage) message2).getValues().get(0)); } else if (event.equals(HCatConstants.HCAT_DROP_PARTITION_EVENT)) { Assert.assertEquals("topic://hcat.mydb.mytbl", msg.getJMSDestination() @@ -184,7 +204,8 @@ Assert.assertEquals("mydb", message2.getDB()); Assert.assertEquals("mytbl", ((DropPartitionMessage) message2).getTable()); Assert.assertEquals(1, ((DropPartitionMessage) message2).getPartitions().size()); - Assert.assertEquals("2011", ((DropPartitionMessage) message2).getPartitions().get(0).get("b")); + Assert.assertEquals("2011", ((DropPartitionMessage) message2).getPartitions().get(0).get( + "b")); } else if (event.equals(HCatConstants.HCAT_DROP_TABLE_EVENT)) { Assert.assertEquals("topic://hcat.mydb", msg.getJMSDestination().toString()); @@ -199,11 +220,20 @@ Assert.assertEquals("topic://" + HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX, msg .getJMSDestination().toString()); - DropDatabaseMessage message = deserializer.getDropDatabaseMessage(messageBody); + DropDatabaseMessage message = deserializer.getDropDatabaseMessage(messageBody); Assert.assertEquals("mydb", message.getDB()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof DropDatabaseMessage); Assert.assertEquals("mydb", message2.getDB()); + } else if (event.equals(HCatConstants.HCAT_ALTER_TABLE_EVENT)) { + Assert.assertEquals("topic://hcat.mydb", msg.getJMSDestination().toString()); + AlterTableMessage message = deserializer.getAlterTableMessage(messageBody); + Assert.assertEquals("mytbl", message.getTable()); + Assert.assertEquals("mydb", message.getDB()); + HCatEventMessage message2 = MessagingUtils.getMessage(msg); + Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterTableMessage); + Assert.assertEquals("mydb", message2.getDB()); + Assert.assertEquals("mytbl", ((AlterTableMessage) message2).getTable()); } else if (event.equals(HCatConstants.HCAT_PARTITION_DONE_EVENT)) { // TODO: Fill in when PARTITION_DONE_EVENT is supported. Assert.assertTrue("Unexpected: HCAT_PARTITION_DONE_EVENT not supported (yet).", false); Index: hcatalog/src/test/e2e/templeton/README.txt =================================================================== --- hcatalog/src/test/e2e/templeton/README.txt (revision 1653459) +++ hcatalog/src/test/e2e/templeton/README.txt (working copy) @@ -223,3 +223,14 @@ Adding Tests ------------ ToDo: add some guidelines + +Running on Tez +1. set up Tez as in http://tez.apache.org/install.html +2. set hive.execution.engine=tez in hive-site.xml (actually is this needed?) +3. add hive.execution.engine=tez to templeton.hive.properties in webhcat-site.xml +4. add to mapred-env.sh/yarn-env.sh (as you defined these in step 1) +export TEZ_VERSION=0.5.3 +export TEZ_JARS=/Users/ekoifman/dev/apache-tez-client-${TEZ_VERSION} +export TEZ_CONF_DIR=${TEZ_JARS}/conf +export HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*:${HADOOP_CLASSPATH} +(w/o this you'll see something like "java.lang.NoClassDefFoundError: org/apache/tez/dag/api/SessionNotRunning") Index: hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh =================================================================== --- hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh (revision 1653459) +++ hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh (working copy) @@ -48,5 +48,17 @@ ${HADOOP_HOME}/bin/hadoop fs -put /Users/ekoifman/dev/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz /apps/templeton/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz ${HADOOP_HOME}/bin/hadoop fs -put /Users/ekoifman/dev/mysql-connector-java-5.1.30/mysql-connector-java-5.1.30-bin.jar /apps/templeton/jdbc/mysql-connector-java.jar + +#Tez set up (http://tez.apache.org/install.html) +#if not using Tez - ignore this +${HADOOP_HOME}/bin/hdfs dfs -put /Users/ekoifman/dev/apache-tez-${TEZ_VERSION}-src/tez-dist/target/tez-${TEZ_VERSION}.tar.gz /apps/tez-${TEZ_VERSION}.tar.gz +${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp/tezin +${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp/tezout +${HADOOP_HOME}/bin/hdfs dfs -put /Users/ekoifman/dev/hive/build.sh /tmp/tezin +#Above line is for Sanity Check: this is to run #6 in http://tez.apache.org/install.html +#$HADOOP_HOME/bin/hadoop jar tez-examples-0.5.3.jar orderedwordcount /tmp/tezin /tmp/tezout + + + #check what got deployed -${HADOOP_HOME}/bin/hdfs dfs -ls -R /apps/templeton webhcate2e /user/templeton /user/hive/warehouse +${HADOOP_HOME}/bin/hdfs dfs -ls -R /apps webhcate2e /user/templeton /user/hive/warehouse Index: hcatalog/src/test/e2e/templeton/deployers/env.sh =================================================================== --- hcatalog/src/test/e2e/templeton/deployers/env.sh (revision 1653459) +++ hcatalog/src/test/e2e/templeton/deployers/env.sh (working copy) @@ -22,15 +22,30 @@ # define necessary env vars here and source it in other files -export HADOOP_VERSION=2.4.1-SNAPSHOT -#export HIVE_VERSION=0.14.0-SNAPSHOT -export PIG_VERSION=0.12.2-SNAPSHOT +echo ${HADOOP_VERSION}; +if [ -z ${HADOOP_VERSION} ]; then + export HADOOP_VERSION=2.4.1-SNAPSHOT +fi + +if [ -z ${HIVE_VERSION} ]; then + export HIVE_VERSION=0.14.0-SNAPSHOT +fi + +if [ -z ${PIG_VERSION} ]; then + export PIG_VERSION=0.12.2-SNAPSHOT +fi + #Root of project source tree -export PROJ_HOME=/Users/${USER}/dev/hive +if [ -z ${PROJ_HOME} ]; then + export PROJ_HOME=/Users/${USER}/dev/hive +fi export HIVE_HOME=${PROJ_HOME}/packaging/target/apache-hive-${HIVE_VERSION}-bin/apache-hive-${HIVE_VERSION}-bin -export HADOOP_HOME=/Users/${USER}/dev/hwxhadoop/hadoop-dist/target/hadoop-${HADOOP_VERSION} +if [ -z ${HADOOP_HOME} ]; then + export HADOOP_HOME=/Users/${USER}/dev/hwxhadoop/hadoop-dist/target/hadoop-${HADOOP_VERSION} +fi + #Make sure Pig is built for the Hadoop version you are running export PIG_TAR_PATH=/Users/${USER}/dev/pig-${PIG_VERSION}-src/build #this is part of Pig distribution Index: hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm =================================================================== --- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (revision 1653459) +++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (working copy) @@ -878,7 +878,7 @@ if (defined($testCmd->{'check_job_percent_complete'})) { my $pcValue = $res_hash->{'percentComplete'}; my $expectedPercentComplete = $testCmd->{'check_job_percent_complete'}; - if ( (!defined $pcValue) || $pcValue ne $expectedPercentComplete ) { + if ( (!defined $pcValue) || $pcValue !~ m/$expectedPercentComplete/ ) { print $log "check_job_percent_complete failed. got percentComplete $pcValue, expected $expectedPercentComplete"; $result = 0; } Index: hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf =================================================================== --- hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (revision 1653459) +++ hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (working copy) @@ -405,7 +405,7 @@ { #test select a,b - 'num' => 7, + 'num' => 7,#seems to be the same as test 6 except for percent_complete check 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=select count(*) from mynums', ], @@ -414,7 +414,7 @@ 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', - 'check_job_percent_complete' => 'map 100% reduce 100%', + 'check_job_percent_complete' => 'map 100% reduce 100%|100% complete', 'check_job_exit_value' => 0, 'check_call_back' => 1, Index: hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java =================================================================== --- hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java (revision 1653459) +++ hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java (working copy) @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hive.hcatalog.common.HCatException; @@ -467,6 +468,31 @@ public abstract String getMessageBusTopicName(String dbName, String tableName) throws HCatException; /** + * Get a list of notifications + * @param lastEventId The last event id that was consumed by this reader. The returned + * notifications will start at the next eventId available this eventId that + * matches the filter. + * @param maxEvents Maximum number of events to return. If < 1, then all available events will + * be returned. + * @param filter Filter to determine if message should be accepted. If null, then all + * available events up to maxEvents will be returned. + * @return list of notifications, sorted by eventId. It is guaranteed that the events are in + * the order that the operations were done on the database. + * @throws HCatException + */ + public abstract List getNextNotification(long lastEventId, + int maxEvents, + IMetaStoreClient.NotificationFilter filter) + throws HCatException; + + /** + * Get the most recently used notification id. + * @return + * @throws HCatException + */ + public abstract long getCurrentNotificationEventId() throws HCatException; + + /** * Close the hcatalog client. * * @throws HCatException Index: hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java =================================================================== --- hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (revision 1653459) +++ hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (working copy) @@ -23,20 +23,25 @@ import java.util.List; import java.util.Map; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -825,7 +830,8 @@ @Override public String getMessageBusTopicName(String dbName, String tableName) throws HCatException { try { - return hmsClient.getTable(dbName, tableName).getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME); + return hmsClient.getTable(dbName, tableName).getParameters().get( + HCatConstants.HCAT_MSGBUS_TOPIC_NAME); } catch (MetaException e) { throw new HCatException("MetaException while retrieving JMS Topic name.", e); @@ -836,8 +842,37 @@ "TException while retrieving JMS Topic name.", e); } } - + @Override + public List getNextNotification(long lastEventId, int maxEvents, + IMetaStoreClient.NotificationFilter filter) + throws HCatException { + try { + List events = new ArrayList(); + NotificationEventResponse rsp = hmsClient.getNextNotification(lastEventId, maxEvents, filter); + if (rsp != null && rsp.getEvents() != null) { + for (NotificationEvent event : rsp.getEvents()) { + events.add(new HCatNotificationEvent(event)); + } + } + return events; + } catch (TException e) { + throw new ConnectionFailureException("TException while getting notifications", e); + } + } + + @Override + public long getCurrentNotificationEventId() throws HCatException { + try { + CurrentNotificationEventId id = hmsClient.getCurrentNotificationEventId(); + return id.getEventId(); + } catch (TException e) { + throw new ConnectionFailureException("TException while getting current notification event " + + "id " , e); + } + } + + @Override public String serializeTable(HCatTable hcatTable) throws HCatException { return MetadataSerializer.get().serializeTable(hcatTable); } @@ -905,8 +940,10 @@ @Override public HCatPartitionSpec deserializePartitionSpec(List hcatPartitionSpecStrings) throws HCatException { - HCatPartitionSpec hcatPartitionSpec = MetadataSerializer.get().deserializePartitionSpec(hcatPartitionSpecStrings); - hcatPartitionSpec.hcatTable(getTable(hcatPartitionSpec.getDbName(), hcatPartitionSpec.getTableName())); + HCatPartitionSpec hcatPartitionSpec = MetadataSerializer.get() + .deserializePartitionSpec(hcatPartitionSpecStrings); + hcatPartitionSpec + .hcatTable(getTable(hcatPartitionSpec.getDbName(), hcatPartitionSpec.getTableName())); return hcatPartitionSpec; } } Index: hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatNotificationEvent.java =================================================================== --- hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatNotificationEvent.java (revision 0) +++ hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatNotificationEvent.java (working copy) @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.api; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; + +/** + * A wrapper class for {@link org.apache.hadoop.hive.metastore.api.NotificationEvent}, + * so that if that class changes we can still keep this one constant for backward compatibility + */ +public class HCatNotificationEvent { + private long eventId; + private int eventTime; + private String eventType; + private String dbName; + private String tableName; + private String message; + + HCatNotificationEvent(NotificationEvent event) { + eventId = event.getEventId(); + eventTime = event.getEventTime(); + eventType = event.getEventType(); + dbName = event.getDbName(); + tableName = event.getTableName(); + message = event.getMessage(); + } + + public long getEventId() { + return eventId; + } + + public int getEventTime() { + return eventTime; + } + + public String getEventType() { + return eventType; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("eventId:"); + buf.append(eventId); + buf.append(" eventTime:"); + buf.append(eventTime); + buf.append(" eventType:<"); + buf.append(eventType); + buf.append("> dbName:<"); + buf.append(dbName); + buf.append("> tableName:<"); + buf.append(tableName); + buf.append("> message:<"); + buf.append(message); + buf.append(">"); + return buf.toString(); + } +} Index: hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java (revision 1653459) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java (working copy) @@ -91,18 +91,25 @@ * */ public void run() { - FileSystem fs = null; while (!stop) { try { // Put each check in a separate try/catch, so if that particular // cycle fails, it'll try again on the next cycle. + FileSystem fs=null; try { - if (fs == null) { - fs = new Path(storage_root).getFileSystem(appConf); - } + fs = new Path(storage_root).getFileSystem(appConf); checkFiles(fs); } catch (Exception e) { LOG.error("Cleanup cycle failed: " + e.getMessage()); + } finally { + if(fs != null) { + try { + fs.close(); + } + catch (Exception e) { + LOG.error("Closing file system failed: " + e.getMessage()); + } + } } long sleepMillis = (long) (Math.random() * interval); Index: hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (revision 1653459) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (working copy) @@ -42,6 +42,8 @@ import javax.ws.rs.core.UriBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -57,6 +59,8 @@ * General utility methods. */ public class TempletonUtils { + private static final Log LOG = LogFactory.getLog(TempletonUtils.class); + /** * Is the object non-empty? */ @@ -98,6 +102,24 @@ public static final Pattern PIG_COMPLETE = Pattern.compile(" \\d+% complete$"); //looking for map = 100%, reduce = 100% public static final Pattern HIVE_COMPLETE = Pattern.compile(" map = (\\d+%),\\s+reduce = (\\d+%).*$"); + /** + * Hive on Tez produces progress report that looks like this + * Map 1: -/- Reducer 2: 0/1 + * Map 1: -/- Reducer 2: 0(+1)/1 + * Map 1: -/- Reducer 2: 1/1 + * + * -/- means there are no tasks (yet) + * 0/1 means 1 total tasks, 0 completed + * 1(+2)/3 means 3 total, 1 completed and 2 running + * + * HIVE-8495, in particular https://issues.apache.org/jira/secure/attachment/12675504/Screen%20Shot%202014-10-16%20at%209.35.26%20PM.png + * has more examples. + * To report progress, we'll assume all tasks are equal size and compute "completed" as percent of "total" + * "(Map|Reducer) (\\d+:) ((-/-)|(\\d+(\\(\\+\\d+\\))?/\\d+))" is the complete pattern but we'll drop "-/-" to exclude + * groups that don't add information such as "Map 1: -/-" + */ + public static final Pattern TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:) (\\d+(\\(\\+\\d+\\))?/\\d+)"); + public static final Pattern TEZ_COUNTERS = Pattern.compile("\\d+"); /** * Extract the percent complete line from Pig or Jar jobs. @@ -115,6 +137,31 @@ if(hive.find()) { return "map " + hive.group(1) + " reduce " + hive.group(2); } + Matcher tez = TEZ_COMPLETE.matcher(line); + if(tez.find()) { + int totalTasks = 0; + int completedTasks = 0; + do { + //here each group looks something like "Map 2: 2/4" "Reducer 3: 1(+2)/4" + //just parse the numbers and ignore one from "Map 2" and from "(+2)" if it's there + Matcher counts = TEZ_COUNTERS.matcher(tez.group()); + List items = new ArrayList(4); + while(counts.find()) { + items.add(counts.group()); + } + completedTasks += Integer.parseInt(items.get(1)); + if(items.size() == 3) { + totalTasks += Integer.parseInt(items.get(2)); + } + else { + totalTasks += Integer.parseInt(items.get(3)); + } + } while(tez.find()); + if(totalTasks == 0) { + return "0% complete (0 total tasks)"; + } + return completedTasks * 100 / totalTasks + "% complete"; + } return null; } Index: itests/hcatalog-unit/pom.xml =================================================================== --- itests/hcatalog-unit/pom.xml (revision 1653459) +++ itests/hcatalog-unit/pom.xml (working copy) @@ -66,6 +66,12 @@ test + org.apache.hive.hcatalog + hive-webhcat-java-client + ${project.version} + test + + org.apache.hive hive-hbase-handler ${project.version} Index: itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java =================================================================== --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java (revision 0) +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java (working copy) @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.api; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; + +import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hive.hcatalog.common.HCatConstants; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.listener.DbNotificationListener; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This can't use TestHCatClient because it has to have control over certain conf variables when + * the metastore is started. Plus, we don't need a metastore running in another thread. The + * local one is fine. + */ +public class TestHCatClientNotification { + + private static final Log LOG = LogFactory.getLog(TestHCatClientNotification.class.getName()); + private static HCatClient hCatClient; + private int startTime; + private long firstEventId; + + @BeforeClass + public static void setupClient() throws Exception { + HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, + DbNotificationListener.class.getName()); + hCatClient = HCatClient.create(conf); + } + + @Before + public void setup() throws Exception { + long now = System.currentTimeMillis() / 1000; + startTime = 0; + if (now > Integer.MAX_VALUE) fail("Bummer, time has fallen over the edge"); + else startTime = (int)now; + firstEventId = hCatClient.getCurrentNotificationEventId(); + } + + @Test + public void createDatabase() throws Exception { + hCatClient.createDatabase(HCatCreateDBDesc.create("myhcatdb").build()); + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, events.size()); + + HCatNotificationEvent event = events.get(0); + assertEquals(firstEventId + 1, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_CREATE_DATABASE_EVENT, event.getEventType()); + assertNull(event.getDbName()); + assertNull(event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"CREATE_DATABASE\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"myhcatdb\",\"timestamp\":[0-9]+}")); + } + + @Test + public void dropDatabase() throws Exception { + String dbname = "hcatdropdb"; + hCatClient.createDatabase(HCatCreateDBDesc.create(dbname).build()); + hCatClient.dropDatabase(dbname, false, HCatClient.DropDBMode.RESTRICT); + + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(2, events.size()); + + HCatNotificationEvent event = events.get(1); + assertEquals(firstEventId + 2, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_DROP_DATABASE_EVENT, event.getEventType()); + assertEquals(dbname, event.getDbName()); + assertNull(event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_DATABASE\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"hcatdropdb\",\"timestamp\":[0-9]+}")); + } + + @Test + public void createTable() throws Exception { + String dbName = "default"; + String tableName = "hcatcreatetable"; + HCatTable table = new HCatTable(dbName, tableName); + table.cols(Arrays.asList(new HCatFieldSchema("onecol", TypeInfoFactory.stringTypeInfo, ""))); + hCatClient.createTable(HCatCreateTableDesc.create(table).build()); + + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, events.size()); + + HCatNotificationEvent event = events.get(0); + assertEquals(firstEventId + 1, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_CREATE_TABLE_EVENT, event.getEventType()); + assertEquals(dbName, event.getDbName()); + assertNull(event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"CREATE_TABLE\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"hcatcreatetable\",\"timestamp\":[0-9]+}")); + } + + // TODO - Currently no way to test alter table, as this interface doesn't support alter table + + @Test + public void dropTable() throws Exception { + String dbName = "default"; + String tableName = "hcatdroptable"; + HCatTable table = new HCatTable(dbName, tableName); + table.cols(Arrays.asList(new HCatFieldSchema("onecol", TypeInfoFactory.stringTypeInfo, ""))); + hCatClient.createTable(HCatCreateTableDesc.create(table).build()); + hCatClient.dropTable(dbName, tableName, false); + + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(2, events.size()); + + HCatNotificationEvent event = events.get(1); + assertEquals(firstEventId + 2, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_DROP_TABLE_EVENT, event.getEventType()); + assertEquals(dbName, event.getDbName()); + assertEquals(tableName, event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_TABLE\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + + "\"hcatdroptable\",\"timestamp\":[0-9]+}")); + } + + @Test + public void addPartition() throws Exception { + String dbName = "default"; + String tableName = "hcataddparttable"; + String partColName = "pc"; + HCatTable table = new HCatTable(dbName, tableName); + table.partCol(new HCatFieldSchema(partColName, TypeInfoFactory.stringTypeInfo, "")); + table.cols(Arrays.asList(new HCatFieldSchema("onecol", TypeInfoFactory.stringTypeInfo, ""))); + hCatClient.createTable(HCatCreateTableDesc.create(table).build()); + String partName = "testpart"; + Map partSpec = new HashMap(1); + partSpec.put(partColName, partName); + hCatClient.addPartition( + HCatAddPartitionDesc.create( + new HCatPartition(table, partSpec, null) + ).build() + ); + + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(2, events.size()); + + HCatNotificationEvent event = events.get(1); + assertEquals(firstEventId + 2, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType()); + assertEquals("default", event.getDbName()); + assertEquals(tableName, event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"ADD_PARTITION\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + + "\"hcataddparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"pc\":\"testpart\"}]}")); + } + + // TODO - currently no way to test alter partition, as HCatClient doesn't support it. + @Test + public void dropPartition() throws Exception { + String dbName = "default"; + String tableName = "hcatdropparttable"; + String partColName = "pc"; + HCatTable table = new HCatTable(dbName, tableName); + table.partCol(new HCatFieldSchema(partColName, TypeInfoFactory.stringTypeInfo, "")); + table.cols(Arrays.asList(new HCatFieldSchema("onecol", TypeInfoFactory.stringTypeInfo, ""))); + hCatClient.createTable(HCatCreateTableDesc.create(table).build()); + String partName = "testpart"; + Map partSpec = new HashMap(1); + partSpec.put(partColName, partName); + hCatClient.addPartition( + HCatAddPartitionDesc.create( + new HCatPartition(table, partSpec, null) + ).build() + ); + hCatClient.dropPartitions(dbName, tableName, partSpec, false); + + List events = hCatClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, events.size()); + + HCatNotificationEvent event = events.get(2); + assertEquals(firstEventId + 3, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT, event.getEventType()); + assertEquals("default", event.getDbName()); + assertEquals(tableName, event.getTableName()); + assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_PARTITION\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + + "\"hcatdropparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"pc\":\"testpart\"}]}")); + } + + @Test + public void getOnlyMaxEvents() throws Exception { + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatdb1").build()); + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatdb2").build()); + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatdb3").build()); + + List events = hCatClient.getNextNotification(firstEventId, 2, null); + assertEquals(2, events.size()); + assertEquals(firstEventId + 1, events.get(0).getEventId()); + assertEquals(firstEventId + 2, events.get(1).getEventId()); + } + + @Test + public void filter() throws Exception { + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatf1").build()); + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatf2").build()); + hCatClient.dropDatabase("hcatf2", false, HCatClient.DropDBMode.RESTRICT); + + IMetaStoreClient.NotificationFilter filter = new IMetaStoreClient.NotificationFilter() { + @Override + public boolean accept(NotificationEvent event) { + return event.getEventType().equals(HCatConstants.HCAT_DROP_DATABASE_EVENT); + } + }; + List events = hCatClient.getNextNotification(firstEventId, 0, filter); + assertEquals(1, events.size()); + assertEquals(firstEventId + 3, events.get(0).getEventId()); + } + + @Test + public void filterWithMax() throws Exception { + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatm1").build()); + hCatClient.createDatabase(HCatCreateDBDesc.create("hcatm2").build()); + hCatClient.dropDatabase("hcatm2", false, HCatClient.DropDBMode.RESTRICT); + + IMetaStoreClient.NotificationFilter filter = new IMetaStoreClient.NotificationFilter() { + @Override + public boolean accept(NotificationEvent event) { + return event.getEventType().equals(HCatConstants.HCAT_CREATE_DATABASE_EVENT); + } + }; + List events = hCatClient.getNextNotification(firstEventId, 1, filter); + assertEquals(1, events.size()); + assertEquals(firstEventId + 1, events.get(0).getEventId()); + } +} Index: itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java =================================================================== --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java (revision 1653459) +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java (working copy) @@ -138,8 +138,6 @@ "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"mytable\",\"timestamp\":[0-9]+}")); } - // TODO After HIVE-9175 is committed - /* @Test public void alterTable() throws Exception { List cols = new ArrayList(); @@ -147,12 +145,12 @@ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters); - Table table = new Table("alttable", "default", "me", startTime, startTime, 0, sd, null, - emptyParameters, null, null, null); + Table table = new Table("alttable", "default", "me", startTime, startTime, 0, sd, + new ArrayList(), emptyParameters, null, null, null); msClient.createTable(table); - table = new Table("alttable", "default", "me", startTime, startTime + 1, 0, sd, null, - emptyParameters, null, null, null); + table = new Table("alttable", "default", "me", startTime, startTime + 1, 0, sd, + new ArrayList(), emptyParameters, null, null, null); msClient.alter_table("default", "alttable", table); NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); @@ -168,7 +166,6 @@ "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"alttable\"," + "\"timestamp\":[0-9]+}")); } - */ @Test public void dropTable() throws Exception { @@ -228,6 +225,41 @@ } @Test + public void alterPartition() throws Exception { + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "nocomment")); + List partCols = new ArrayList(); + partCols.add(new FieldSchema("ds", "string", "")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, + serde, null, null, emptyParameters); + Table table = new Table("alterparttable", "default", "me", startTime, startTime, 0, sd, + partCols, emptyParameters, null, null, null); + msClient.createTable(table); + + Partition partition = new Partition(Arrays.asList("today"), "default", "alterparttable", + startTime, startTime, sd, emptyParameters); + msClient.add_partition(partition); + + Partition newPart = new Partition(Arrays.asList("today"), "default", "alterparttable", + startTime, startTime + 1, sd, emptyParameters); + msClient.alter_partition("default", "alterparttable", newPart); + + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + + NotificationEvent event = rsp.getEvents().get(2); + assertEquals(firstEventId + 3, event.getEventId()); + assertTrue(event.getEventTime() >= startTime); + assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType()); + assertEquals("default", event.getDbName()); + assertEquals("alterparttable", event.getTableName()); + assertTrue(event.getMessage().matches( "\\{\"eventType\":\"ALTER_PARTITION\",\"server\":\"\"," + + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"alterparttable\"," + + "\"timestamp\":[0-9]+,\"values\":\\[\"today\"]}")); + } + + @Test public void dropPartition() throws Exception { List cols = new ArrayList(); cols.add(new FieldSchema("col1", "int", "nocomment")); Index: itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java =================================================================== --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java (revision 1653459) +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java (working copy) @@ -205,169 +205,218 @@ @Test - public void testStaticPartition() throws Exception { - String tableName = "staticpart"; - CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string, part2 string)"); - Assert.assertEquals(0,ret.getResponseCode()); + public void testInsertNonPartTable() throws Exception { + //case 1 is non-partitioned table. + String tableName = "nonpart"; + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string)"); + Assert.assertEquals(0, ret.getResponseCode()); + + String tableLoc = warehouseDir + "/" + tableName; assertExistence(warehouseDir + "/" + tableName); + + //case1A: insert into non-partitioned table. setPermission(warehouseDir + "/" + tableName); + ret = driver.run("insert into table " + tableName + " select key,value from mysrc"); + Assert.assertEquals(0, ret.getResponseCode()); - ret = driver.run("insert into table " + tableName + " partition(part1='1', part2='1') select key,value from mysrc where part1='1' and part2='1'"); - Assert.assertEquals(0,ret.getResponseCode()); + verifyPermission(warehouseDir + "/" + tableName); + Assert.assertTrue(listStatus(tableLoc).size() > 0); + for (String child : listStatus(tableLoc)) { + verifyPermission(child); + } - verifyPermission(warehouseDir + "/" + tableName + "/part1=1"); - verifyPermission(warehouseDir + "/" + tableName + "/part1=1/part2=1"); + //case1B: insert overwrite non-partitioned-table + setPermission(warehouseDir + "/" + tableName, 1); + ret = driver.run("insert overwrite table " + tableName + " select key,value from mysrc"); + Assert.assertEquals(0, ret.getResponseCode()); - Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1").size() > 0); - for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1")) { - verifyPermission(child); + verifyPermission(warehouseDir + "/" + tableName, 1); + Assert.assertTrue(listStatus(tableLoc).size() > 0); + for (String child : listStatus(tableLoc)) { + verifyPermission(child, 1); } } @Test - public void testAlterPartition() throws Exception { - String tableName = "alterpart"; - CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 int, part2 int, part3 int)"); - Assert.assertEquals(0,ret.getResponseCode()); + public void testInsertStaticSinglePartition() throws Exception { + String tableName = "singlestaticpart"; + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string)"); + Assert.assertEquals(0, ret.getResponseCode()); assertExistence(warehouseDir + "/" + tableName); setPermission(warehouseDir + "/" + tableName); - ret = driver.run("insert into table " + tableName + " partition(part1='1',part2='1',part3='1') select key,value from mysrc"); - Assert.assertEquals(0,ret.getResponseCode()); + //insert into test + ret = driver.run("insert into table " + tableName + " partition(part1='1') select key,value from mysrc where part1='1' and part2='1'"); + Assert.assertEquals(0, ret.getResponseCode()); - assertExistence(warehouseDir + "/" + tableName); + verifyPermission(warehouseDir + "/" + tableName); + verifyPermission(warehouseDir + "/" + tableName + "/part1=1"); + + Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=1").size() > 0); + for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=1")) { + verifyPermission(child); + } + + //insert overwrite test setPermission(warehouseDir + "/" + tableName, 1); + ret = driver.run("insert overwrite table " + tableName + " partition(part1='1') select key,value from mysrc where part1='1' and part2='1'"); + Assert.assertEquals(0, ret.getResponseCode()); - //alter partition - ret = driver.run("alter table " + tableName + " partition (part1='1',part2='1',part3='1') rename to partition (part1='2',part2='2',part3='2')"); - Assert.assertEquals(0,ret.getResponseCode()); + verifyPermission(warehouseDir + "/" + tableName, 1); + verifyPermission(warehouseDir + "/" + tableName + "/part1=1", 1); - verifyPermission(warehouseDir + "/" + tableName + "/part1=2", 1); - verifyPermission(warehouseDir + "/" + tableName + "/part1=2/part2=2", 1); - verifyPermission(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2", 1); - - Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2").size() > 0); - for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2")) { + Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=1").size() > 0); + for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=1")) { verifyPermission(child, 1); } } - @Test - public void testDynamicPartitions() throws Exception { - String tableName = "dynamicpart"; - + public void testInsertStaticDualPartition() throws Exception { + String tableName = "dualstaticpart"; CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string, part2 string)"); - Assert.assertEquals(0,ret.getResponseCode()); + Assert.assertEquals(0, ret.getResponseCode()); assertExistence(warehouseDir + "/" + tableName); setPermission(warehouseDir + "/" + tableName); - ret = driver.run("insert into table " + tableName + " partition (part1,part2) select key,value,part1,part2 from mysrc"); - Assert.assertEquals(0,ret.getResponseCode()); + //insert into test + ret = driver.run("insert into table " + tableName + " partition(part1='1', part2='1') select key,value from mysrc where part1='1' and part2='1'"); + Assert.assertEquals(0, ret.getResponseCode()); + verifyPermission(warehouseDir + "/" + tableName); verifyPermission(warehouseDir + "/" + tableName + "/part1=1"); verifyPermission(warehouseDir + "/" + tableName + "/part1=1/part2=1"); - verifyPermission(warehouseDir + "/" + tableName + "/part1=2"); - verifyPermission(warehouseDir + "/" + tableName + "/part1=2/part2=2"); - Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1").size() > 0); for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1")) { verifyPermission(child); } - Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2").size() > 0); - for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2")) { - verifyPermission(child); + //insert overwrite test + setPermission(warehouseDir + "/" + tableName, 1); + ret = driver.run("insert overwrite table " + tableName + " partition(part1='1', part2='1') select key,value from mysrc where part1='1' and part2='1'"); + Assert.assertEquals(0, ret.getResponseCode()); + + verifyPermission(warehouseDir + "/" + tableName, 1); + verifyPermission(warehouseDir + "/" + tableName + "/part1=1", 1); + verifyPermission(warehouseDir + "/" + tableName + "/part1=1/part2=1", 1); + + Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1").size() > 0); + for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=1/part2=1")) { + verifyPermission(child, 1); } } @Test - public void testExternalTable() throws Exception { - String tableName = "externaltable"; + public void testInsertDualDynamicPartitions() throws Exception { + String tableName = "dualdynamicpart"; - String myLocation = warehouseDir + "/myfolder"; - FileSystem fs = FileSystem.get(new URI(myLocation), conf); - fs.mkdirs(new Path(myLocation)); - setPermission(myLocation); + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string, part2 string)"); + Assert.assertEquals(0, ret.getResponseCode()); + assertExistence(warehouseDir + "/" + tableName); - CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) LOCATION '" + myLocation + "'"); + //Insert into test, with permission set 0. + setPermission(warehouseDir + "/" + tableName, 0); + ret = driver.run("insert into table " + tableName + " partition (part1,part2) select key,value,part1,part2 from mysrc"); + Assert.assertEquals(0, ret.getResponseCode()); + + verifyDualPartitionTable(warehouseDir + "/" + tableName, 0); + + //Insert overwrite test, with permission set 1. + setPermission(warehouseDir + "/" + tableName, 1); + ret = driver.run("insert overwrite table " + tableName + " partition (part1,part2) select key,value,part1,part2 from mysrc"); + Assert.assertEquals(0, ret.getResponseCode()); + + verifyDualPartitionTable(warehouseDir + "/" + tableName, 1); + } + + @Test + public void testInsertSingleDynamicPartition() throws Exception { + String tableName = "singledynamicpart"; + + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string)"); Assert.assertEquals(0,ret.getResponseCode()); + String tableLoc = warehouseDir + "/" + tableName; + assertExistence(tableLoc); - ret = driver.run("insert into table " + tableName + " select key,value from mysrc"); + //Insert into test, with permission set 0. + setPermission(tableLoc, 0); + ret = driver.run("insert into table " + tableName + " partition (part1) select key,value,part1 from mysrc"); Assert.assertEquals(0,ret.getResponseCode()); + verifySinglePartition(tableLoc, 0); - Assert.assertTrue(listStatus(myLocation).size() > 0); - for (String child : listStatus(myLocation)) { - verifyPermission(child); - } + //Insert overwrite test, with permission set 1. + setPermission(tableLoc, 1); + ret = driver.run("insert overwrite table " + tableName + " partition (part1) select key,value,part1 from mysrc"); + Assert.assertEquals(0,ret.getResponseCode()); + verifySinglePartition(tableLoc, 1); + + //delete and re-insert using insert overwrite. There's different code paths insert vs insert overwrite for new tables. + ret = driver.run("DROP TABLE " + tableName); + Assert.assertEquals(0, ret.getResponseCode()); + ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 string)"); + Assert.assertEquals(0, ret.getResponseCode()); + + assertExistence(warehouseDir + "/" + tableName); + setPermission(warehouseDir + "/" + tableName); + + ret = driver.run("insert overwrite table " + tableName + " partition (part1) select key,value,part1 from mysrc"); + Assert.assertEquals(0, ret.getResponseCode()); + + verifySinglePartition(tableLoc, 0); } @Test - public void testInsert() throws Exception { - //case 1 is non-partitioned table. - String tableName = "insert"; - - CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string)"); + public void testAlterPartition() throws Exception { + String tableName = "alterpart"; + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 int, part2 int, part3 int)"); Assert.assertEquals(0,ret.getResponseCode()); - String tableLoc = warehouseDir + "/" + tableName; assertExistence(warehouseDir + "/" + tableName); + setPermission(warehouseDir + "/" + tableName); - //case1A: insert into non-partitioned table. - setPermission(warehouseDir + "/" + tableName); - ret = driver.run("insert into table " + tableName + " select key,value from mysrc"); + ret = driver.run("insert into table " + tableName + " partition(part1='1',part2='1',part3='1') select key,value from mysrc"); Assert.assertEquals(0,ret.getResponseCode()); - Assert.assertTrue(listStatus(tableLoc).size() > 0); - for (String child : listStatus(tableLoc)) { - verifyPermission(child); - } + assertExistence(warehouseDir + "/" + tableName); + setPermission(warehouseDir + "/" + tableName, 1); - //case1B: insert overwrite non-partitioned-table - setPermission(warehouseDir + "/" + tableName, 1); - ret = driver.run("insert overwrite table " + tableName + " select key,value from mysrc"); + //alter partition + ret = driver.run("alter table " + tableName + " partition (part1='1',part2='1',part3='1') rename to partition (part1='2',part2='2',part3='2')"); Assert.assertEquals(0,ret.getResponseCode()); - Assert.assertTrue(listStatus(tableLoc).size() > 0); - for (String child : listStatus(tableLoc)) { + verifyPermission(warehouseDir + "/" + tableName + "/part1=2", 1); + verifyPermission(warehouseDir + "/" + tableName + "/part1=2/part2=2", 1); + verifyPermission(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2", 1); + + Assert.assertTrue(listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2").size() > 0); + for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2")) { verifyPermission(child, 1); } + } - //case 2 is partitioned table. - tableName = "insertpartition"; + @Test + public void testExternalTable() throws Exception { + String tableName = "externaltable"; - ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 int, part2 int)"); - Assert.assertEquals(0,ret.getResponseCode()); + String myLocation = warehouseDir + "/myfolder"; + FileSystem fs = FileSystem.get(new URI(myLocation), conf); + fs.mkdirs(new Path(myLocation)); + setPermission(myLocation); - ret = driver.run("insert overwrite table " + tableName + " partition(part1='1',part2='1') select key,value from mysrc"); + CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) LOCATION '" + myLocation + "'"); Assert.assertEquals(0,ret.getResponseCode()); - String partLoc = warehouseDir + "/" + tableName + "/part1=1/part2=1"; - assertExistence(partLoc); - - //case 2A: insert into partitioned table. - setPermission(partLoc); - ret = driver.run("insert overwrite table " + tableName + " partition(part1='1',part2='1') select key,value from mysrc"); + ret = driver.run("insert into table " + tableName + " select key,value from mysrc"); Assert.assertEquals(0,ret.getResponseCode()); - Assert.assertTrue(listStatus(partLoc).size() > 0); - for (String child : listStatus(partLoc)) { + Assert.assertTrue(listStatus(myLocation).size() > 0); + for (String child : listStatus(myLocation)) { verifyPermission(child); } - - //case 2B: insert into non-partitioned table. - setPermission(partLoc, 1); - ret = driver.run("insert overwrite table " + tableName + " partition(part1='1',part2='1') select key,value from mysrc"); - Assert.assertEquals(0,ret.getResponseCode()); - - Assert.assertTrue(listStatus(tableLoc).size() > 0); - for (String child : listStatus(partLoc)) { - verifyPermission(child, 1); - } } @Test @@ -422,7 +471,7 @@ } //case 2B: insert data overwrite into non-partitioned table. - setPermission(partLoc, 1); + setPermission(tableLoc, 1); ret = driver.run("LOAD DATA LOCAL INPATH '" + dataFilePath + "' OVERWRITE INTO TABLE " + tableName + " PARTITION (part1='1',part2='1')"); Assert.assertEquals(0,ret.getResponseCode()); @@ -487,7 +536,7 @@ } //case 2B: insert data overwrite into non-partitioned table. - setPermission(partLoc, 1); + setPermission(tableLoc, 1); fs.copyFromLocalFile(dataFilePath, new Path(location)); ret = driver.run("LOAD DATA INPATH '" + location + "' OVERWRITE INTO TABLE " + tableName + " PARTITION (part1='1',part2='1')"); Assert.assertEquals(0,ret.getResponseCode()); @@ -596,6 +645,40 @@ } } + private void verifySinglePartition(String tableLoc, int index) throws Exception { + verifyPermission(tableLoc + "/part1=1", index); + verifyPermission(tableLoc + "/part1=2", index); + + Assert.assertTrue(listStatus(tableLoc + "/part1=1").size() > 0); + for (String child : listStatus(tableLoc + "/part1=1")) { + verifyPermission(child, index); + } + + Assert.assertTrue(listStatus(tableLoc + "/part1=2").size() > 0); + for (String child : listStatus(tableLoc + "/part1=2")) { + verifyPermission(child, index); + } + } + + private void verifyDualPartitionTable(String baseTablePath, int index) throws Exception { + verifyPermission(baseTablePath, index); + verifyPermission(baseTablePath + "/part1=1", index); + verifyPermission(baseTablePath + "/part1=1/part2=1", index); + + verifyPermission(baseTablePath + "/part1=2", index); + verifyPermission(baseTablePath + "/part1=2/part2=2", index); + + Assert.assertTrue(listStatus(baseTablePath + "/part1=1/part2=1").size() > 0); + for (String child : listStatus(baseTablePath + "/part1=1/part2=1")) { + verifyPermission(child, index); + } + + Assert.assertTrue(listStatus(baseTablePath + "/part1=2/part2=2").size() > 0); + for (String child : listStatus(baseTablePath + "/part1=2/part2=2")) { + verifyPermission(child, index); + } + } + private void assertExistence(String locn) throws Exception { Assert.assertTrue(fs.exists(new Path(locn))); } Index: itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java =================================================================== --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java (revision 1653459) +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; +import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser; public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; @@ -406,6 +407,56 @@ assertTrue(flattenedSql.contains(parentTab)); } + /** + * Test script formatting + * @throws Exception + */ + public void testPostgresFilter() throws Exception { + String testScript[] = { + "-- this is a comment", + "DROP TABLE IF EXISTS fooTab;", + HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT + ";", + "CREATE TABLE fooTab(id INTEGER);", + "DROP TABLE footab;", + "-- ending comment" + }; + + String expectedScriptWithOptionPresent[] = { + "DROP TABLE IF EXISTS fooTab", + HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT, + "CREATE TABLE fooTab(id INTEGER)", + "DROP TABLE footab", + }; + + NestedScriptParser noDbOptParser = HiveSchemaHelper + .getDbCommandParser("postgres"); + String expectedSQL = StringUtils.join( + expectedScriptWithOptionPresent, System.getProperty("line.separator")) + + System.getProperty("line.separator"); + File testScriptFile = generateTestScript(testScript); + String flattenedSql = noDbOptParser.buildCommand( + testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + assertEquals(expectedSQL, flattenedSql); + + String expectedScriptWithOptionAbsent[] = { + "DROP TABLE IF EXISTS fooTab", + "CREATE TABLE fooTab(id INTEGER)", + "DROP TABLE footab", + }; + + NestedScriptParser dbOptParser = HiveSchemaHelper.getDbCommandParser( + "postgres", + PostgresCommandParser.POSTGRES_SKIP_STANDARD_STRINGS_DBOPT, + null, null, null); + expectedSQL = StringUtils.join( + expectedScriptWithOptionAbsent, System.getProperty("line.separator")) + + System.getProperty("line.separator"); + testScriptFile = generateTestScript(testScript); + flattenedSql = dbOptParser.buildCommand( + testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + assertEquals(expectedSQL, flattenedSql); + } + private File generateTestScript(String [] stmts) throws IOException { File testScriptFile = File.createTempFile("schematest", ".sql"); testScriptFile.deleteOnExit(); Index: itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java =================================================================== --- itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (revision 1653459) +++ itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (working copy) @@ -131,7 +131,7 @@ } builder.append("127.0.1.1,"); builder.append(InetAddress.getLocalHost().getCanonicalHostName()); - conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), + conf.setStrings(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(superUserShortName), builder.toString()); } @@ -294,7 +294,7 @@ private void setGroupsInConf(String[] groupNames, String proxyUserName) throws IOException { conf.set( - DefaultImpersonationProvider.getProxySuperuserGroupConfKey(proxyUserName), + DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(proxyUserName), StringUtils.join(",", Arrays.asList(groupNames))); configureSuperUserIPAddresses(conf, proxyUserName); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); Index: itests/qtest/pom.xml =================================================================== --- itests/qtest/pom.xml (revision 1653459) +++ itests/qtest/pom.xml (working copy) @@ -435,7 +435,7 @@ templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm" queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/" queryFile="${qfile}" - excludeQueryFile="${minimr.query.files},${minitez.query.files}" + excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files}" queryFileRegex="${qfile_regex}" clusterMode="${clustermode}" runDisabled="${run_disabled}" @@ -516,6 +516,24 @@ hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> + + Index: itests/src/test/resources/testconfiguration.properties =================================================================== --- itests/src/test/resources/testconfiguration.properties (revision 1653459) +++ itests/src/test/resources/testconfiguration.properties (working copy) @@ -140,6 +140,7 @@ parallel.q,\ ptf.q,\ sample1.q,\ + selectDistinctStar.q,\ script_env_var1.q,\ script_env_var2.q,\ script_pipe.q,\ @@ -270,7 +271,11 @@ auto_sortmerge_join_5.q,\ auto_sortmerge_join_7.q,\ auto_sortmerge_join_8.q,\ - auto_sortmerge_join_9.q + auto_sortmerge_join_9.q,\ + auto_join30.q,\ + auto_join21.q,\ + auto_join29.q,\ + auto_join_filters.q minitez.query.files=bucket_map_join_tez1.q,\ @@ -297,6 +302,14 @@ vectorized_dynamic_partition_pruning.q,\ tez_multi_union.q +encrypted.query.files=encryption_join_unencrypted_tbl.q,\ + encryption_insert_partition_static.q,\ + encryption_insert_partition_dynamic.q,\ + encryption_join_with_different_encryption_keys.q,\ + encryption_select_read_only_encrypted_tbl.q,\ + encryption_select_read_only_unencrypted_tbl.q,\ + encryption_load_data_to_encrypted_tables.q + beeline.positive.exclude=add_part_exist.q,\ alter1.q,\ alter2.q,\ Index: itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1653459) +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -38,8 +38,9 @@ import java.io.PrintStream; import java.io.Serializable; import java.io.StringWriter; -import java.lang.System; +import java.lang.RuntimeException; import java.net.URL; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,11 +50,13 @@ import java.util.List; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import junit.framework.Assert; +import org.apache.commons.lang.StringUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; @@ -75,6 +78,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; @@ -84,6 +88,10 @@ import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.processors.CommandProcessor; +import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.processors.HiveCommand; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -103,10 +111,16 @@ public class QTestUtil { public static final String UTF_8 = "UTF-8"; + + // security property names + private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; + private static final String CRLF = System.getProperty("line.separator"); + private static final Log LOG = LogFactory.getLog("QTestUtil"); private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private final String defaultInitScript = "q_test_init.sql"; private final String defaultCleanupScript = "q_test_cleanup.sql"; + private final String[] testOnlyCommands = new String[]{"crypto"}; private String testWarehouse; private final String testFiles; @@ -131,6 +145,8 @@ private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; + private HadoopShims.HdfsEncryptionShim hes = null; + private boolean miniMr = false; private String hadoopVer = null; private QTestSetup setup = null; private boolean isSessionStateStarted = false; @@ -255,7 +271,7 @@ // Plug verifying metastore in for testing. conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, - "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); + "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); if (mr != null) { assert dfs != null; @@ -282,6 +298,7 @@ mr, tez, spark, + encrypted, none; public static MiniClusterType valueForString(String type) { @@ -291,6 +308,8 @@ return tez; } else if (type.equals("spark")) { return spark; + } else if (type.equals("encrypted")) { + return encrypted; } else { return none; } @@ -303,6 +322,15 @@ this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript); } + private String getKeyProviderURI() { + // Use the target directory if it is not specified + String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); + String keyDir = HIVE_ROOT + "ql/target/"; + + // put the jks file in the current test path only for test purpose + return "jceks://file" + new Path(keyDir, "test.jks").toUri(); + } + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript) throws Exception { @@ -327,8 +355,25 @@ int numberOfDataNodes = 4; if (clusterType != MiniClusterType.none && clusterType != MiniClusterType.spark) { - dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); - FileSystem fs = dfs.getFileSystem(); + FileSystem fs = null; + + if (clusterType == MiniClusterType.encrypted) { + // Set the security key provider so that the MiniDFS cluster is initialized + // with encryption + conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); + + dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); + fs = dfs.getFileSystem(); + + // set up the java key provider for encrypted hdfs cluster + hes = shims.createHdfsEncryptionShim(fs, conf); + + LOG.info("key provider is initialized"); + } else { + dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); + fs = dfs.getFileSystem(); + } + String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { mr = shims.getMiniTezCluster(conf, 4, uriString, 1); @@ -344,7 +389,6 @@ if (dataDir == null) { dataDir = new File(".").getAbsolutePath() + "/data/files"; } - testFiles = dataDir; // Use the current directory if it is not specified @@ -369,7 +413,10 @@ } public void shutdown() throws Exception { - cleanUp(); + if (System.getenv(QTEST_LEAVE_FILES) == null) { + cleanUp(); + } + setup.tearDown(); if (mr != null) { mr.shutdown(); @@ -576,6 +623,19 @@ setup.postTest(conf); } + public void clearKeysCreatedInTests() { + if (hes == null) { + return; + } + try { + for (String keyAlias : hes.getKeys()) { + hes.deleteKey(keyAlias); + } + } catch (IOException e) { + LOG.error("Fail to clean the keys created in test due to the error", e); + } + } + /** * Clear out any side effects of running tests */ @@ -646,12 +706,17 @@ } clearTablesCreatedDuringTests(); + clearKeysCreatedInTests(); - // allocate and initialize a new conf since a test can - // modify conf by using 'set' commands - conf = new HiveConf (Driver.class); - initConf(); - db = Hive.get(conf); // propagate new conf to meta store + if (clusterType != MiniClusterType.encrypted) { + // allocate and initialize a new conf since a test can + // modify conf by using 'set' commands + conf = new HiveConf (Driver.class); + initConf(); + // renew the metastore since the cluster type is unencrypted + db = Hive.get(conf); // propagate new conf to meta store + } + setup.preTest(conf); } @@ -664,6 +729,7 @@ } clearTablesCreatedDuringTests(); + clearKeysCreatedInTests(); SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); @@ -864,23 +930,131 @@ try { return drv.run(qMap.get(tname)).getResponseCode(); } catch (CommandNeedRetryException e) { - // TODO Auto-generated catch block + LOG.error("driver failed to run the command: " + tname + " due to the exception: ", e); e.printStackTrace(); return -1; } } - private static final String CRLF = System.getProperty("line.separator"); public int executeClient(String tname1, String tname2) { - String commands = getCommands(tname1) + CRLF + getCommands(tname2); - return cliDriver.processLine(commands); + String commands = getCommand(tname1) + CRLF + getCommand(tname2); + return executeClientInternal(commands); } public int executeClient(String tname) { - return cliDriver.processLine(getCommands(tname)); + return executeClientInternal(getCommand(tname)); } - private String getCommands(String tname) { + private int executeClientInternal(String commands) { + String [] cmds = commands.split(";"); + int rc = 0; + + String command = ""; + for (String oneCmd : cmds) { + if (StringUtils.endsWith(oneCmd, "\\")) { + command += StringUtils.chop(oneCmd) + "\\;"; + continue; + } else { + if (isHiveCommand(oneCmd)) { + command = oneCmd; + } else { + command += oneCmd; + } + } + if (StringUtils.isBlank(command)) { + continue; + } + + if (isCommandUsedForTesting(command)) { + rc = executeTestCommand(command); + } else { + rc = cliDriver.processLine(command); + } + + if (rc != 0) { + break; + } + command = ""; + } + + return rc; + } + + private boolean isHiveCommand(String command) { + String[] cmd = command.trim().split("\\s+"); + if (HiveCommand.find(cmd) != null) { + return true; + } else if (HiveCommand.find(cmd, HiveCommand.ONLY_FOR_TESTING) != null) { + return true; + } else { + return false; + } + } + + private int executeTestCommand(final String command) { + String commandName = command.trim().split("\\s+")[0]; + String commandArgs = command.trim().substring(commandName.length()); + + if (commandArgs.endsWith(";")) { + commandArgs = StringUtils.chop(commandArgs); + } + + //replace ${hiveconf:hive.metastore.warehouse.dir} with actual dir if existed. + //we only want the absolute path, so remove the header, such as hdfs://localhost:57145 + String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE) + .replaceAll("^[a-zA-Z]+://.*?:\\d+", ""); + commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}", + wareHouseDir); + + enableTestOnlyCmd(SessionState.get().getConf()); + + try { + CommandProcessor proc = getTestCommand(commandName); + if (proc != null) { + CommandProcessorResponse response = proc.run(commandArgs.trim()); + + int rc = response.getResponseCode(); + if (rc != 0) { + SessionState.get().out.println(response); + } + + return rc; + } else { + throw new RuntimeException("Could not get CommandProcessor for command: " + commandName); + } + } catch (Exception e) { + throw new RuntimeException("Could not execute test command: " + e.getMessage()); + } + } + + private CommandProcessor getTestCommand(final String commandName) throws SQLException { + HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING); + + if (testCommand == null) { + return null; + } + + return CommandProcessorFactory + .getForHiveCommandInternal(new String[]{commandName}, SessionState.get().getConf(), + testCommand.isOnlyForTesting()); + } + + private void enableTestOnlyCmd(HiveConf conf){ + StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST)); + for(String c : testOnlyCommands){ + securityCMDs.append(","); + securityCMDs.append(c); + } + conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString()); + } + + private boolean isCommandUsedForTesting(final String command) { + String commandName = command.trim().split("\\s+")[0]; + HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING); + return testCommand != null; + } + + private String getCommand(String tname) { String commands = qMap.get(tname); StringBuilder newCommands = new StringBuilder(commands.length()); int lastMatchEnd = 0; @@ -895,6 +1069,11 @@ return commands; } + private boolean isComment(final String line) { + String lineTrimmed = line.trim(); + return lineTrimmed.startsWith("#") || lineTrimmed.startsWith("--"); + } + public boolean shouldBeSkipped(String tname) { return qSkipSet.contains(tname); } @@ -919,7 +1098,7 @@ // Move all data from dest4_sequencefile to dest4 drv - .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); + .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); // Drop dest4_sequencefile db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", @@ -1171,6 +1350,7 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { String maskPattern = "#### A masked pattern was here ####"; + String partialMaskPattern = "#### A PARTIAL masked pattern was here ####"; String line; BufferedReader in; @@ -1184,11 +1364,26 @@ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8")); boolean lastWasMasked = false; + boolean partialMaskWasMatched = false; + Matcher matcher; while (null != (line = in.readLine())) { - for (Pattern pattern : patterns) { - line = pattern.matcher(line).replaceAll(maskPattern); + if (clusterType == MiniClusterType.encrypted) { + for (Pattern pattern : partialReservedPlanMask) { + matcher = pattern.matcher(line); + if (matcher.find()) { + line = partialMaskPattern + " " + matcher.group(0); + partialMaskWasMatched = true; + break; + } + } } + if (!partialMaskWasMatched) { + for (Pattern pattern : patterns) { + line = pattern.matcher(line).replaceAll(maskPattern); + } + } + if (line.equals(maskPattern)) { // We're folding multiple masked lines into one. if (!lastWasMasked) { @@ -1200,6 +1395,7 @@ out.write(line); out.write("\n"); lastWasMasked = false; + partialMaskWasMatched = false; } } @@ -1239,9 +1435,15 @@ ".*DagName:.*", ".*Input:.*/data/files/.*", ".*Output:.*/data/files/.*", - ".*total number of created files now is.*" + ".*total number of created files now is.*", + ".*.hive-staging.*" }); + private final Pattern[] partialReservedPlanMask = toPattern(new String[] { + "data/warehouse/(.*?/)+\\.hive-staging" // the directory might be db/table/partition + //TODO: add more expected test result here + }); + public int checkCliDriverResults(String tname) throws Exception { assert(qMap.containsKey(tname)); @@ -1467,7 +1669,7 @@ zooKeeper.close(); } - int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); + int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS); zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() { @Override public void process(WatchedEvent arg0) { @@ -1492,6 +1694,8 @@ } public void tearDown() throws Exception { + CuratorFrameworkSingleton.closeAndReleaseInstance(); + if (zooKeeperCluster != null) { zooKeeperCluster.shutdown(); zooKeeperCluster = null; @@ -1636,8 +1840,10 @@ } public static void outputTestFailureHelpMessage() { - System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " - + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."); + System.err.println( + "See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " + + "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " + + "test cases logs."); System.err.flush(); } Index: itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java =================================================================== --- itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java (revision 1653459) +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java (working copy) @@ -17,8 +17,10 @@ */ package org.apache.hadoop.hive.ql.hooks; +import java.util.Arrays; import java.io.IOException; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.session.SessionState; @@ -30,7 +32,7 @@ for (WriteEntity output : hookContext.getOutputs()) { Path tableLocation = new Path(output.getTable().getDataLocation().toString()); FileSystem fs = tableLocation.getFileSystem(SessionState.get().getConf()); - assert(fs.listStatus(tableLocation).length == 0); + assert(fs.listStatus(tableLocation, FileUtils.HIDDEN_FILES_PATH_FILTER).length == 0); } } } Index: metastore/scripts/upgrade/derby/020-HIVE-9296.derby.sql =================================================================== --- metastore/scripts/upgrade/derby/020-HIVE-9296.derby.sql (revision 1653459) +++ metastore/scripts/upgrade/derby/020-HIVE-9296.derby.sql (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE "APP"."NOTIFICATION_LOG" ("NL_ID" BIGINT NOT NULL, "DB_NAME" VARCHAR(128), "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, "MESSAGE" VARCHAR(1024) NOT NULL, "TBL_NAME" VARCHAR(128)); +CREATE TABLE "APP"."NOTIFICATION_LOG" ("NL_ID" BIGINT NOT NULL, "DB_NAME" VARCHAR(128), "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, "MESSAGE" LONG VARCHAR, "TBL_NAME" VARCHAR(128)); CREATE TABLE "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID" BIGINT NOT NULL, "NEXT_EVENT_ID" BIGINT NOT NULL); Index: metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql =================================================================== --- metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql (revision 1653459) +++ metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql (working copy) @@ -104,7 +104,7 @@ CREATE TABLE "APP"."FUNC_RU" ("FUNC_ID" BIGINT NOT NULL, "RESOURCE_TYPE" INTEGER NOT NULL, "RESOURCE_URI" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL); -CREATE TABLE "APP"."NOTIFICATION_LOG" ("NL_ID" BIGINT NOT NULL, "DB_NAME" VARCHAR(128), "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, "MESSAGE" VARCHAR(1024) NOT NULL, "TBL_NAME" VARCHAR(128)); +CREATE TABLE "APP"."NOTIFICATION_LOG" ("NL_ID" BIGINT NOT NULL, "DB_NAME" VARCHAR(128), "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, "MESSAGE" LONG VARCHAR, "TBL_NAME" VARCHAR(128)); CREATE TABLE "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID" BIGINT NOT NULL, "NEXT_EVENT_ID" BIGINT NOT NULL); Index: metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql =================================================================== --- metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql (revision 1653459) +++ metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql (working copy) @@ -6,7 +6,7 @@ EVENT_TYPE varchar(32) NOT NULL, DB_NAME varchar(128) NULL, TBL_NAME varchar(128) NULL, - MESSAGE varchar(1024) NOT NULL + MESSAGE text NULL ); GO Index: metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql =================================================================== --- metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql (revision 1653459) +++ metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql (working copy) @@ -578,7 +578,7 @@ EVENT_TYPE varchar(32) NOT NULL, DB_NAME varchar(128) NULL, TBL_NAME varchar(128) NULL, - MESSAGE varchar(1024) NOT NULL + MESSAGE text NULL ); ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID); Index: metastore/scripts/upgrade/mysql/020-HIVE-9296.mysql.sql =================================================================== --- metastore/scripts/upgrade/mysql/020-HIVE-9296.mysql.sql (revision 1653459) +++ metastore/scripts/upgrade/mysql/020-HIVE-9296.mysql.sql (working copy) @@ -6,7 +6,7 @@ `EVENT_TYPE` varchar(32) NOT NULL, `DB_NAME` varchar(128), `TBL_NAME` varchar(128), - `MESSAGE` varchar(1024) NOT NULL, + `MESSAGE` mediumtext, PRIMARY KEY (`NL_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; Index: metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql =================================================================== --- metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql (revision 1653459) +++ metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql (working copy) @@ -803,7 +803,7 @@ `EVENT_TYPE` varchar(32) NOT NULL, `DB_NAME` varchar(128), `TBL_NAME` varchar(128), - `MESSAGE` varchar(1024) NOT NULL, + `MESSAGE` mediumtext, PRIMARY KEY (`NL_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; Index: metastore/scripts/upgrade/oracle/021-HIVE-9296.oracle.sql =================================================================== --- metastore/scripts/upgrade/oracle/021-HIVE-9296.oracle.sql (revision 1653459) +++ metastore/scripts/upgrade/oracle/021-HIVE-9296.oracle.sql (working copy) @@ -6,7 +6,7 @@ EVENT_TYPE VARCHAR2(32) NOT NULL, DB_NAME VARCHAR2(128), TBL_NAME VARCHAR2(128), - MESSAGE VARCHAR2(1024) NOT NULL + MESSAGE CLOB NULL ); ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID); Index: metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql =================================================================== --- metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql (revision 1653459) +++ metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql (working copy) @@ -559,7 +559,7 @@ EVENT_TYPE VARCHAR2(32) NOT NULL, DB_NAME VARCHAR2(128), TBL_NAME VARCHAR2(128), - MESSAGE VARCHAR2(1024) NOT NULL + MESSAGE CLOB NULL ); ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID); Index: metastore/scripts/upgrade/postgres/020-HIVE-9296.postgres.sql =================================================================== --- metastore/scripts/upgrade/postgres/020-HIVE-9296.postgres.sql (revision 1653459) +++ metastore/scripts/upgrade/postgres/020-HIVE-9296.postgres.sql (working copy) @@ -6,7 +6,7 @@ "EVENT_TYPE" VARCHAR(32) NOT NULL, "DB_NAME" VARCHAR(128), "TBL_NAME" VARCHAR(128), - "MESSAGE" VARCHAR(1024) NOT NULL, + "MESSAGE" text, PRIMARY KEY ("NL_ID") ); Index: metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql =================================================================== --- metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql (revision 1653459) +++ metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql (working copy) @@ -587,7 +587,7 @@ "EVENT_TYPE" VARCHAR(32) NOT NULL, "DB_NAME" VARCHAR(128), "TBL_NAME" VARCHAR(128), - "MESSAGE" VARCHAR(1024) NOT NULL, + "MESSAGE" text, PRIMARY KEY ("NL_ID") ); Index: metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java =================================================================== --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java (revision 1653459) +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java (working copy) @@ -184,7 +184,7 @@ __this__skewedColValueLocationMaps.put(__this__skewedColValueLocationMaps_copy_key, __this__skewedColValueLocationMaps_copy_value); } - this.skewedColValueLocationMaps = __this__skewedColValueLocationMaps; + this.skewedColValueLocationMaps = __this__skewedColValueLocationMaps; } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1653459) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -3125,9 +3125,14 @@ oldPart = alterHandler.alterPartition(getMS(), wh, db_name, tbl_name, part_vals, new_part); + // Only fetch the table if we actually have a listener + Table table = null; for (MetaStoreEventListener listener : listeners) { + if (table == null) { + table = getMS().getTable(db_name, tbl_name); + } AlterPartitionEvent alterPartitionEvent = - new AlterPartitionEvent(oldPart, new_part, true, this); + new AlterPartitionEvent(oldPart, new_part, table, true, this); alterPartitionEvent.setEnvironmentContext(envContext); listener.onAlterPartition(alterPartitionEvent); } @@ -3178,6 +3183,8 @@ oldParts = alterHandler.alterPartitions(getMS(), wh, db_name, tbl_name, new_parts); Iterator olditr = oldParts.iterator(); + // Only fetch the table if we have a listener that needs it. + Table table = null; for (Partition tmpPart : new_parts) { Partition oldTmpPart = null; if (olditr.hasNext()) { @@ -3187,8 +3194,11 @@ throw new InvalidOperationException("failed to alterpartitions"); } for (MetaStoreEventListener listener : listeners) { + if (table == null) { + table = getMS().getTable(db_name, tbl_name); + } AlterPartitionEvent alterPartitionEvent = - new AlterPartitionEvent(oldTmpPart, tmpPart, true, this); + new AlterPartitionEvent(oldTmpPart, tmpPart, table, true, this); listener.onAlterPartition(alterPartitionEvent); } } @@ -5367,126 +5377,74 @@ // Transaction and locking methods @Override public GetOpenTxnsResponse get_open_txns() throws TException { - try { - return getTxnHandler().getOpenTxns(); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().getOpenTxns(); } // Transaction and locking methods @Override public GetOpenTxnsInfoResponse get_open_txns_info() throws TException { - try { - return getTxnHandler().getOpenTxnsInfo(); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().getOpenTxnsInfo(); } @Override public OpenTxnsResponse open_txns(OpenTxnRequest rqst) throws TException { - try { - return getTxnHandler().openTxns(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().openTxns(rqst); } @Override public void abort_txn(AbortTxnRequest rqst) throws NoSuchTxnException, TException { - try { - getTxnHandler().abortTxn(rqst); - } catch (MetaException e) { - throw new TException(e); - } + getTxnHandler().abortTxn(rqst); } @Override public void commit_txn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { - try { - getTxnHandler().commitTxn(rqst); - } catch (MetaException e) { - throw new TException(e); - } + getTxnHandler().commitTxn(rqst); } @Override public LockResponse lock(LockRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { - try { - return getTxnHandler().lock(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().lock(rqst); } @Override public LockResponse check_lock(CheckLockRequest rqst) throws NoSuchTxnException, TxnAbortedException, NoSuchLockException, TException { - try { - return getTxnHandler().checkLock(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().checkLock(rqst); } @Override public void unlock(UnlockRequest rqst) throws NoSuchLockException, TxnOpenException, TException { - try { - getTxnHandler().unlock(rqst); - } catch (MetaException e) { - throw new TException(e); - } + getTxnHandler().unlock(rqst); } @Override public ShowLocksResponse show_locks(ShowLocksRequest rqst) throws TException { - try { - return getTxnHandler().showLocks(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().showLocks(rqst); } @Override public void heartbeat(HeartbeatRequest ids) throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, TException { - try { - getTxnHandler().heartbeat(ids); - } catch (MetaException e) { - throw new TException(e); - } + getTxnHandler().heartbeat(ids); } @Override public HeartbeatTxnRangeResponse heartbeat_txn_range(HeartbeatTxnRangeRequest rqst) throws TException { - try { - return getTxnHandler().heartbeatTxnRange(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().heartbeatTxnRange(rqst); } @Override public void compact(CompactionRequest rqst) throws TException { - try { - getTxnHandler().compact(rqst); - } catch (MetaException e) { - throw new TException(e); - } + getTxnHandler().compact(rqst); } @Override public ShowCompactResponse show_compact(ShowCompactRequest rqst) throws TException { - try { - return getTxnHandler().showCompact(rqst); - } catch (MetaException e) { - throw new TException(e); - } + return getTxnHandler().showCompact(rqst); } @Override Index: metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java (revision 1653459) +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java (working copy) @@ -20,17 +20,20 @@ import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; public class AlterPartitionEvent extends ListenerEvent { private final Partition oldPart; private final Partition newPart; + private final Table table; - public AlterPartitionEvent(Partition oldPart, Partition newPart, + public AlterPartitionEvent(Partition oldPart, Partition newPart, Table table, boolean status, HMSHandler handler) { super(status, handler); this.oldPart = oldPart; this.newPart = newPart; + this.table = table; } /** @@ -47,4 +50,12 @@ public Partition getNewPartition() { return newPart; } + + /** + * Get the table this partition is in + * @return + */ + public Table getTable() { + return table; + } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java (revision 1653459) +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java (working copy) @@ -52,51 +52,58 @@ * or runAs set since these are only potential compactions not actual ones. */ public Set findPotentialCompactions(int maxAborted) throws MetaException { - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Connection dbConn = null; Set response = new HashSet(); Statement stmt = null; try { - stmt = dbConn.createStatement(); - // Check for completed transactions - String s = "select distinct ctc_database, ctc_table, " + + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + // Check for completed transactions + String s = "select distinct ctc_database, ctc_table, " + "ctc_partition from COMPLETED_TXN_COMPONENTS"; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - while (rs.next()) { - CompactionInfo info = new CompactionInfo(); - info.dbname = rs.getString(1); - info.tableName = rs.getString(2); - info.partName = rs.getString(3); - response.add(info); - } + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + while (rs.next()) { + CompactionInfo info = new CompactionInfo(); + info.dbname = rs.getString(1); + info.tableName = rs.getString(2); + info.partName = rs.getString(3); + response.add(info); + } - // Check for aborted txns - s = "select tc_database, tc_table, tc_partition " + + // Check for aborted txns + s = "select tc_database, tc_table, tc_partition " + "from TXNS, TXN_COMPONENTS " + "where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' " + "group by tc_database, tc_table, tc_partition " + "having count(*) > " + maxAborted; - LOG.debug("Going to execute query <" + s + ">"); - rs = stmt.executeQuery(s); - while (rs.next()) { - CompactionInfo info = new CompactionInfo(); - info.dbname = rs.getString(1); - info.tableName = rs.getString(2); - info.partName = rs.getString(3); - info.tooManyAborts = true; - response.add(info); + LOG.debug("Going to execute query <" + s + ">"); + rs = stmt.executeQuery(s); + while (rs.next()) { + CompactionInfo info = new CompactionInfo(); + info.dbname = rs.getString(1); + info.tableName = rs.getString(2); + info.partName = rs.getString(3); + info.tooManyAborts = true; + response.add(info); + } + + LOG.debug("Going to rollback"); + dbConn.rollback(); + } catch (SQLException e) { + LOG.error("Unable to connect to transaction database " + e.getMessage()); + checkRetryable(dbConn, e, "findPotentialCompactions"); + } finally { + closeDbConn(dbConn); + closeStmt(stmt); } - - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e) { - LOG.error("Unable to connect to transaction database " + e.getMessage()); - } finally { - closeDbConn(dbConn); - closeStmt(stmt); + return response; } - return response; + catch (RetryException e) { + return findPotentialCompactions(maxAborted); + } } /** @@ -107,35 +114,31 @@ */ public void setRunAs(long cq_id, String user) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { - stmt = dbConn.createStatement(); - String s = "update COMPACTION_QUEUE set cq_run_as = '" + user + "' where cq_id = " + cq_id; - LOG.debug("Going to execute update <" + s + ">"); - if (stmt.executeUpdate(s) != 1) { - LOG.error("Unable to update compaction record"); - LOG.debug("Going to rollback"); - dbConn.rollback(); - } - LOG.debug("Going to commit"); - dbConn.commit(); - } catch (SQLException e) { - LOG.error("Unable to update compaction queue, " + e.getMessage()); - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "setRunAs"); - } finally { - closeDbConn(dbConn); - closeStmt(stmt); - } - } catch (DeadlockException e) { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + stmt = dbConn.createStatement(); + String s = "update COMPACTION_QUEUE set cq_run_as = '" + user + "' where cq_id = " + cq_id; + LOG.debug("Going to execute update <" + s + ">"); + if (stmt.executeUpdate(s) != 1) { + LOG.error("Unable to update compaction record"); + LOG.debug("Going to rollback"); + dbConn.rollback(); + } + LOG.debug("Going to commit"); + dbConn.commit(); + } catch (SQLException e) { + LOG.error("Unable to update compaction queue, " + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "setRunAs"); + } finally { + closeDbConn(dbConn); + closeStmt(stmt); + } + } catch (RetryException e) { setRunAs(cq_id, user); - } finally { - deadlockCnt = 0; } } @@ -147,14 +150,15 @@ */ public CompactionInfo findNextToCompact(String workerId) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; CompactionInfo info = new CompactionInfo(); Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); String s = "select cq_id, cq_database, cq_table, cq_partition, " + - "cq_type from COMPACTION_QUEUE where cq_state = '" + INITIATED_STATE + "'"; + "cq_type from COMPACTION_QUEUE where cq_state = '" + INITIATED_STATE + "'"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); if (!rs.next()) { @@ -175,7 +179,7 @@ // Now, update this record as being worked on by this worker. long now = getDbTime(dbConn); s = "update COMPACTION_QUEUE set cq_worker_id = '" + workerId + "', " + - "cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id; + "cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id; LOG.debug("Going to execute update <" + s + ">"); if (stmt.executeUpdate(s) != 1) { LOG.error("Unable to update compaction record"); @@ -187,38 +191,34 @@ return info; } catch (SQLException e) { LOG.error("Unable to select next element for compaction, " + e.getMessage()); - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "findNextToCompact"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "findNextToCompact"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { return findNextToCompact(workerId); - } finally { - deadlockCnt = 0; } } /** * This will mark an entry in the queue as compacted * and put it in the ready to clean state. - * @param info info on the compaciton entry to mark as compacted. + * @param info info on the compaction entry to mark as compacted. */ public void markCompacted(CompactionInfo info) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); String s = "update COMPACTION_QUEUE set cq_state = '" + READY_FOR_CLEANING + "', " + - "cq_worker_id = null where cq_id = " + info.id; + "cq_worker_id = null where cq_id = " + info.id; LOG.debug("Going to execute update <" + s + ">"); if (stmt.executeUpdate(s) != 1) { LOG.error("Unable to update compaction record"); @@ -228,23 +228,18 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.error("Unable to update compaction queue " + e.getMessage()); - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "markCompacted"); + LOG.error("Unable to update compaction queue " + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "markCompacted"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { markCompacted(info); - } finally { - deadlockCnt = 0; } } @@ -254,45 +249,48 @@ * @return information on the entry in the queue. */ public List findReadyToClean() throws MetaException { - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Connection dbConn = null; List rc = new ArrayList(); Statement stmt = null; try { - stmt = dbConn.createStatement(); - String s = "select cq_id, cq_database, cq_table, cq_partition, " + + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + String s = "select cq_id, cq_database, cq_table, cq_partition, " + "cq_type, cq_run_as from COMPACTION_QUEUE where cq_state = '" + READY_FOR_CLEANING + "'"; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - while (rs.next()) { - CompactionInfo info = new CompactionInfo(); - info.id = rs.getLong(1); - info.dbname = rs.getString(2); - info.tableName = rs.getString(3); - info.partName = rs.getString(4); - switch (rs.getString(5).charAt(0)) { - case MAJOR_TYPE: info.type = CompactionType.MAJOR; break; - case MINOR_TYPE: info.type = CompactionType.MINOR; break; - default: throw new MetaException("Unexpected compaction type " + rs.getString(5)); + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + while (rs.next()) { + CompactionInfo info = new CompactionInfo(); + info.id = rs.getLong(1); + info.dbname = rs.getString(2); + info.tableName = rs.getString(3); + info.partName = rs.getString(4); + switch (rs.getString(5).charAt(0)) { + case MAJOR_TYPE: info.type = CompactionType.MAJOR; break; + case MINOR_TYPE: info.type = CompactionType.MINOR; break; + default: throw new MetaException("Unexpected compaction type " + rs.getString(5)); + } + info.runAs = rs.getString(6); + rc.add(info); } - info.runAs = rs.getString(6); - rc.add(info); - } - LOG.debug("Going to rollback"); - dbConn.rollback(); - return rc; - } catch (SQLException e) { - LOG.error("Unable to select next element for cleaning, " + e.getMessage()); - try { LOG.debug("Going to rollback"); dbConn.rollback(); - } catch (SQLException e1) { + return rc; + } catch (SQLException e) { + LOG.error("Unable to select next element for cleaning, " + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "findReadyToClean"); + throw new MetaException("Unable to connect to transaction database " + + StringUtils.stringifyException(e)); + } finally { + closeDbConn(dbConn); + closeStmt(stmt); } - throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); - } finally { - closeDbConn(dbConn); - closeStmt(stmt); + } catch (RetryException e) { + return findReadyToClean(); } } @@ -303,9 +301,10 @@ */ public void markCleaned(CompactionInfo info) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id; LOG.debug("Going to execute update <" + s + ">"); @@ -318,20 +317,20 @@ // Remove entries from completed_txn_components as well, so we don't start looking there // again. s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " + - "ctc_table = '" + info.tableName + "'"; + "ctc_table = '" + info.tableName + "'"; if (info.partName != null) { s += " and ctc_partition = '" + info.partName + "'"; } LOG.debug("Going to execute update <" + s + ">"); if (stmt.executeUpdate(s) < 1) { LOG.error("Expected to remove at least one row from completed_txn_components when " + - "marking compaction entry as clean!"); + "marking compaction entry as clean!"); } s = "select txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" + - TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" + - info.tableName + "'"; + TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" + + info.tableName + "'"; if (info.partName != null) s += " and tc_partition = '" + info.partName + "'"; LOG.debug("Going to execute update <" + s + ">"); ResultSet rs = stmt.executeQuery(s); @@ -371,23 +370,18 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.error("Unable to delete from compaction queue " + e.getMessage()); - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "markCleaned"); + LOG.error("Unable to delete from compaction queue " + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "markCleaned"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { markCleaned(info); - } finally { - deadlockCnt = 0; } } @@ -396,13 +390,14 @@ */ public void cleanEmptyAbortedTxns() throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); String s = "select txn_id from TXNS where " + - "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " + - "txn_state = '" + TXN_ABORTED + "'"; + "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " + + "txn_state = '" + TXN_ABORTED + "'"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); Set txnids = new HashSet(); @@ -425,21 +420,16 @@ } catch (SQLException e) { LOG.error("Unable to delete from txns table " + e.getMessage()); LOG.debug("Going to rollback"); - try { - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "cleanEmptyAbortedTxns"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "cleanEmptyAbortedTxns"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { cleanEmptyAbortedTxns(); - } finally { - deadlockCnt = 0; } } @@ -454,13 +444,14 @@ */ public void revokeFromLocalWorkers(String hostname) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '" - + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_worker_id like '" - + hostname + "%'"; + + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_worker_id like '" + + hostname + "%'"; LOG.debug("Going to execute update <" + s + ">"); // It isn't an error if the following returns no rows, as the local workers could have died // with nothing assigned to them. @@ -468,24 +459,19 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.error("Unable to change dead worker's records back to initiated state " + - e.getMessage()); - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "revokeFromLocalWorkers"); + LOG.error("Unable to change dead worker's records back to initiated state " + + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "revokeFromLocalWorkers"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { revokeFromLocalWorkers(hostname); - } finally { - deadlockCnt = 0; } } @@ -500,14 +486,15 @@ */ public void revokeTimedoutWorkers(long timeout) throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); - long latestValidStart = getDbTime(dbConn) - timeout; + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + long latestValidStart = getDbTime(dbConn) - timeout; stmt = dbConn.createStatement(); String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '" - + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_start < " - + latestValidStart; + + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_start < " + + latestValidStart; LOG.debug("Going to execute update <" + s + ">"); // It isn't an error if the following returns no rows, as the local workers could have died // with nothing assigned to them. @@ -515,24 +502,19 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.error("Unable to change dead worker's records back to initiated state " + - e.getMessage()); - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "revokeTimedoutWorkers"); + LOG.error("Unable to change dead worker's records back to initiated state " + + e.getMessage()); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "revokeTimedoutWorkers"); throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); closeStmt(stmt); } - } catch (DeadlockException e) { + } catch (RetryException e) { revokeTimedoutWorkers(timeout); - } finally { - deadlockCnt = 0; } } @@ -543,53 +525,55 @@ * @throws MetaException */ public List findColumnsWithStats(CompactionInfo ci) throws MetaException { - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Connection dbConn = null; Statement stmt = null; ResultSet rs = null; try { - String quote = getIdentifierQuoteString(dbConn); - stmt = dbConn.createStatement(); - StringBuilder bldr = new StringBuilder(); - bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote) + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + String quote = getIdentifierQuoteString(dbConn); + stmt = dbConn.createStatement(); + StringBuilder bldr = new StringBuilder(); + bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote) .append(" FROM ") .append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS")) - .append(quote) + .append(quote) .append(" WHERE ") .append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname) - .append("' AND ").append(quote).append("TABLE_NAME").append(quote) - .append(" = '").append(ci.tableName).append("'"); - if (ci.partName != null) { - bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '") + .append("' AND ").append(quote).append("TABLE_NAME").append(quote) + .append(" = '").append(ci.tableName).append("'"); + if (ci.partName != null) { + bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '") .append(ci.partName).append("'"); - } - String s = bldr.toString(); + } + String s = bldr.toString(); /*String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS") + " WHERE DB_NAME='" + ci.dbname + "' AND TABLE_NAME='" + ci.tableName + "'" + (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'");*/ - LOG.debug("Going to execute <" + s + ">"); - rs = stmt.executeQuery(s); - List columns = new ArrayList(); - while(rs.next()) { - columns.add(rs.getString(1)); - } - LOG.debug("Found columns to update stats: " + columns + " on " + ci.tableName + - (ci.partName == null ? "" : "/" + ci.partName)); - dbConn.commit(); - return columns; - } catch (SQLException e) { - try { + LOG.debug("Going to execute <" + s + ">"); + rs = stmt.executeQuery(s); + List columns = new ArrayList(); + while (rs.next()) { + columns.add(rs.getString(1)); + } + LOG.debug("Found columns to update stats: " + columns + " on " + ci.tableName + + (ci.partName == null ? "" : "/" + ci.partName)); + dbConn.commit(); + return columns; + } catch (SQLException e) { LOG.error("Failed to find columns to analyze stats on for " + ci.tableName + - (ci.partName == null ? "" : "/" + ci.partName), e); - dbConn.rollback(); - } catch (SQLException e1) { - //nothing we can do here + (ci.partName == null ? "" : "/" + ci.partName), e); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "findColumnsWithStats"); + throw new MetaException("Unable to connect to transaction database " + + StringUtils.stringifyException(e)); + } finally { + close(rs, stmt, dbConn); } - throw new MetaException("Unable to connect to transaction database " + - StringUtils.stringifyException(e)); - } finally { - close(rs, stmt, dbConn); + } catch (RetryException ex) { + return findColumnsWithStats(ci); } } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (revision 1653459) +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (working copy) @@ -77,7 +77,7 @@ static final private Log LOG = LogFactory.getLog(TxnHandler.class.getName()); static private DataSource connPool; - private static Boolean lockLock = new Boolean("true"); // Random object to lock on for the lock + private final static Object lockLock = new Object(); // Random object to lock on for the lock // method /** @@ -87,10 +87,13 @@ protected HiveConf conf; protected DatabaseProduct dbProduct; - // Transaction timeout, in milliseconds. + // (End user) Transaction timeout, in milliseconds. private long timeout; private String identifierQuoteString; // quotes to use for quoting tables, where necessary + private final long retryInterval; + private final int retryLimit; + private int retryNum; // DEADLOCK DETECTION AND HANDLING // A note to developers of this class. ALWAYS access HIVE_LOCKS before TXNS to avoid deadlock @@ -125,113 +128,122 @@ timeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS); deadlockCnt = 0; buildJumpTable(); + retryInterval = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); + retryLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.HMSHANDLERATTEMPTS); + } public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException { - // We need to figure out the current transaction number and the list of - // open transactions. To avoid needing a transaction on the underlying - // database we'll look at the current transaction number first. If it - // subsequently shows up in the open list that's ok. - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); - Statement stmt = null; try { - stmt = dbConn.createStatement(); - String s = "select ntxn_next - 1 from NEXT_TXN_ID"; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - if (!rs.next()) { - throw new MetaException("Transaction tables not properly " + + // We need to figure out the current transaction number and the list of + // open transactions. To avoid needing a transaction on the underlying + // database we'll look at the current transaction number first. If it + // subsequently shows up in the open list that's ok. + Connection dbConn = null; + Statement stmt = null; + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + String s = "select ntxn_next - 1 from NEXT_TXN_ID"; + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + if (!rs.next()) { + throw new MetaException("Transaction tables not properly " + "initialized, no record found in next_txn_id"); - } - long hwm = rs.getLong(1); - if (rs.wasNull()) { - throw new MetaException("Transaction tables not properly " + + } + long hwm = rs.getLong(1); + if (rs.wasNull()) { + throw new MetaException("Transaction tables not properly " + "initialized, null record found in next_txn_id"); - } + } - List txnInfo = new ArrayList(); - s = "select txn_id, txn_state, txn_user, txn_host from TXNS"; - LOG.debug("Going to execute query<" + s + ">"); - rs = stmt.executeQuery(s); - while (rs.next()) { - char c = rs.getString(2).charAt(0); - TxnState state; - switch (c) { - case TXN_ABORTED: - state = TxnState.ABORTED; - break; + List txnInfo = new ArrayList(); + s = "select txn_id, txn_state, txn_user, txn_host from TXNS"; + LOG.debug("Going to execute query<" + s + ">"); + rs = stmt.executeQuery(s); + while (rs.next()) { + char c = rs.getString(2).charAt(0); + TxnState state; + switch (c) { + case TXN_ABORTED: + state = TxnState.ABORTED; + break; - case TXN_OPEN: - state = TxnState.OPEN; - break; + case TXN_OPEN: + state = TxnState.OPEN; + break; - default: - throw new MetaException("Unexpected transaction state " + c + + default: + throw new MetaException("Unexpected transaction state " + c + " found in txns table"); + } + txnInfo.add(new TxnInfo(rs.getLong(1), state, rs.getString(3), rs.getString(4))); } - txnInfo.add(new TxnInfo(rs.getLong(1), state, rs.getString(3), rs.getString(4))); - } - LOG.debug("Going to rollback"); - dbConn.rollback(); - return new GetOpenTxnsInfoResponse(hwm, txnInfo); - } catch (SQLException e) { - try { LOG.debug("Going to rollback"); dbConn.rollback(); - } catch (SQLException e1) { + return new GetOpenTxnsInfoResponse(hwm, txnInfo); + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "getOpenTxnsInfo"); + throw new MetaException("Unable to select from transaction database: " + getMessage(e) + + StringUtils.stringifyException(e)); + } finally { + closeStmt(stmt); + closeDbConn(dbConn); } - throw new MetaException("Unable to select from transaction database, " - + StringUtils.stringifyException(e)); - } finally { - closeStmt(stmt); - closeDbConn(dbConn); + } catch (RetryException e) { + return getOpenTxnsInfo(); } } public GetOpenTxnsResponse getOpenTxns() throws MetaException { - // We need to figure out the current transaction number and the list of - // open transactions. To avoid needing a transaction on the underlying - // database we'll look at the current transaction number first. If it - // subsequently shows up in the open list that's ok. - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); - Statement stmt = null; try { - timeOutTxns(dbConn); - stmt = dbConn.createStatement(); - String s = "select ntxn_next - 1 from NEXT_TXN_ID"; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - if (!rs.next()) { - throw new MetaException("Transaction tables not properly " + + // We need to figure out the current transaction number and the list of + // open transactions. To avoid needing a transaction on the underlying + // database we'll look at the current transaction number first. If it + // subsequently shows up in the open list that's ok. + Connection dbConn = null; + Statement stmt = null; + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + timeOutTxns(dbConn); + stmt = dbConn.createStatement(); + String s = "select ntxn_next - 1 from NEXT_TXN_ID"; + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + if (!rs.next()) { + throw new MetaException("Transaction tables not properly " + "initialized, no record found in next_txn_id"); - } - long hwm = rs.getLong(1); - if (rs.wasNull()) { - throw new MetaException("Transaction tables not properly " + + } + long hwm = rs.getLong(1); + if (rs.wasNull()) { + throw new MetaException("Transaction tables not properly " + "initialized, null record found in next_txn_id"); - } + } - Set openList = new HashSet(); - s = "select txn_id from TXNS"; - LOG.debug("Going to execute query<" + s + ">"); - rs = stmt.executeQuery(s); - while (rs.next()) { - openList.add(rs.getLong(1)); - } - LOG.debug("Going to rollback"); - dbConn.rollback(); - return new GetOpenTxnsResponse(hwm, openList); - } catch (SQLException e) { - try { + Set openList = new HashSet(); + s = "select txn_id from TXNS"; + LOG.debug("Going to execute query<" + s + ">"); + rs = stmt.executeQuery(s); + while (rs.next()) { + openList.add(rs.getLong(1)); + } LOG.debug("Going to rollback"); dbConn.rollback(); - } catch (SQLException e1) { + return new GetOpenTxnsResponse(hwm, openList); + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "getOpenTxns"); + throw new MetaException("Unable to select from transaction database, " + + StringUtils.stringifyException(e)); + } finally { + closeStmt(stmt); + closeDbConn(dbConn); } - throw new MetaException("Unable to select from transaction database, " - + StringUtils.stringifyException(e)); - } finally { - closeStmt(stmt); - closeDbConn(dbConn); + } catch (RetryException e) { + return getOpenTxns(); } } @@ -259,12 +271,13 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { int numTxns = rqst.getNum_txns(); try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); // Make sure the user has not requested an insane amount of txns. int maxTxns = HiveConf.getIntVar(conf, - HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH); + HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH); if (numTxns > maxTxns) numTxns = maxTxns; stmt = dbConn.createStatement(); @@ -273,7 +286,7 @@ ResultSet rs = stmt.executeQuery(s); if (!rs.next()) { throw new MetaException("Transaction database not properly " + - "configured, can't find next transaction id."); + "configured, can't find next transaction id."); } long first = rs.getLong(1); s = "update NEXT_TXN_ID set ntxn_next = " + (first + numTxns); @@ -281,8 +294,8 @@ stmt.executeUpdate(s); long now = getDbTime(dbConn); s = "insert into TXNS (txn_id, txn_state, txn_started, " + - "txn_last_heartbeat, txn_user, txn_host) values (?, 'o', " + now + ", " + - now + ", '" + rqst.getUser() + "', '" + rqst.getHostname() + "')"; + "txn_last_heartbeat, txn_user, txn_host) values (?, 'o', " + now + ", " + + now + ", '" + rqst.getUser() + "', '" + rqst.getHostname() + "')"; LOG.debug("Going to prepare statement <" + s + ">"); PreparedStatement ps = dbConn.prepareStatement(s); List txnIds = new ArrayList(numTxns); @@ -296,30 +309,26 @@ dbConn.commit(); return new OpenTxnsResponse(txnIds); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "openTxns"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "openTxns"); throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e)); } finally { closeStmt(stmt); closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { return openTxns(rqst); - } finally { - deadlockCnt = 0; } } public void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException { long txnid = rqst.getTxnid(); try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); List txnids = new ArrayList(1); txnids.add(txnid); if (abortTxns(dbConn, txnids) != 1) { @@ -331,31 +340,27 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "abortTxn"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "abortTxn"); throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { abortTxn(rqst); - } finally { - deadlockCnt = 0; } } public void commitTxn(CommitTxnRequest rqst) - throws NoSuchTxnException, TxnAbortedException, MetaException { + throws NoSuchTxnException, TxnAbortedException, MetaException { long txnid = rqst.getTxnid(); try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); // Before we do the commit heartbeat the txn. This is slightly odd in that we're going to // commit it, but it does two things. One, it makes sure the transaction is still valid. @@ -367,11 +372,11 @@ // Move the record from txn_components into completed_txn_components so that the compactor // knows where to look to compact. String s = "insert into COMPLETED_TXN_COMPONENTS select tc_txnid, tc_database, tc_table, " + - "tc_partition from TXN_COMPONENTS where tc_txnid = " + txnid; + "tc_partition from TXN_COMPONENTS where tc_txnid = " + txnid; LOG.debug("Going to execute insert <" + s + ">"); if (stmt.executeUpdate(s) < 1) { LOG.warn("Expected to move at least one record from txn_components to " + - "completed_txn_components when committing txn!"); + "completed_txn_components when committing txn!"); } // Always access TXN_COMPONENTS before HIVE_LOCKS; @@ -388,80 +393,68 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "commitTxn"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "commitTxn"); throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e)); } finally { closeStmt(stmt); closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { commitTxn(rqst); - } finally { - deadlockCnt = 0; } } public LockResponse lock(LockRequest rqst) - throws NoSuchTxnException, TxnAbortedException, MetaException { + throws NoSuchTxnException, TxnAbortedException, MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); return lock(dbConn, rqst, true); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "lock"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "lock"); throw new MetaException("Unable to update transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { return lock(rqst); - } finally { - deadlockCnt = 0; } } public LockResponse lockNoWait(LockRequest rqst) - throws NoSuchTxnException, TxnAbortedException, MetaException { + throws NoSuchTxnException, TxnAbortedException, MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); return lock(dbConn, rqst, false); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "lockNoWait"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "lockNoWait"); throw new MetaException("Unable to update transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { return lockNoWait(rqst); - } finally { - deadlockCnt = 0; } } public LockResponse checkLock(CheckLockRequest rqst) - throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException { + throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); long extLockId = rqst.getLockid(); // Clean up timed out locks timeOutLocks(dbConn); @@ -474,31 +467,27 @@ if (txnid > 0) heartbeatTxn(dbConn, txnid); return checkLock(dbConn, extLockId, true); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "checkLock"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "checkLock"); throw new MetaException("Unable to update transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { return checkLock(rqst); - } finally { - deadlockCnt = 0; } } public void unlock(UnlockRequest rqst) - throws NoSuchLockException, TxnOpenException, MetaException { + throws NoSuchLockException, TxnOpenException, MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); // Odd as it seems, we need to heartbeat first because this touches the // lock table and assures that our locks our still valid. If they are // not, this will throw an exception and the heartbeat will fail. @@ -512,8 +501,8 @@ LOG.debug("Going to rollback"); dbConn.rollback(); String msg = "Unlocking locks associated with transaction" + - " not permitted. Lockid " + extLockId + " is associated with " + - "transaction " + txnid; + " not permitted. Lockid " + extLockId + " is associated with " + + "transaction " + txnid; LOG.error(msg); throw new TxnOpenException(msg); } @@ -529,97 +518,96 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "unlock"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "unlock"); throw new MetaException("Unable to update transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeStmt(stmt); closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { unlock(rqst); - } finally { - deadlockCnt = 0; } } public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException { - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); - ShowLocksResponse rsp = new ShowLocksResponse(); - List elems = new ArrayList(); - Statement stmt = null; try { - stmt = dbConn.createStatement(); + Connection dbConn = null; + ShowLocksResponse rsp = new ShowLocksResponse(); + List elems = new ArrayList(); + Statement stmt = null; + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); - String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " + + String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " + "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host from HIVE_LOCKS"; - LOG.debug("Doing to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - while (rs.next()) { - ShowLocksResponseElement e = new ShowLocksResponseElement(); - e.setLockid(rs.getLong(1)); - long txnid = rs.getLong(2); - if (!rs.wasNull()) e.setTxnid(txnid); - e.setDbname(rs.getString(3)); - e.setTablename(rs.getString(4)); - String partition = rs.getString(5); - if (partition != null) e.setPartname(partition); - switch (rs.getString(6).charAt(0)) { - case LOCK_ACQUIRED: e.setState(LockState.ACQUIRED); break; - case LOCK_WAITING: e.setState(LockState.WAITING); break; - default: throw new MetaException("Unknown lock state " + rs.getString(6).charAt(0)); + LOG.debug("Doing to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + while (rs.next()) { + ShowLocksResponseElement e = new ShowLocksResponseElement(); + e.setLockid(rs.getLong(1)); + long txnid = rs.getLong(2); + if (!rs.wasNull()) e.setTxnid(txnid); + e.setDbname(rs.getString(3)); + e.setTablename(rs.getString(4)); + String partition = rs.getString(5); + if (partition != null) e.setPartname(partition); + switch (rs.getString(6).charAt(0)) { + case LOCK_ACQUIRED: e.setState(LockState.ACQUIRED); break; + case LOCK_WAITING: e.setState(LockState.WAITING); break; + default: throw new MetaException("Unknown lock state " + rs.getString(6).charAt(0)); + } + switch (rs.getString(7).charAt(0)) { + case LOCK_SEMI_SHARED: e.setType(LockType.SHARED_WRITE); break; + case LOCK_EXCLUSIVE: e.setType(LockType.EXCLUSIVE); break; + case LOCK_SHARED: e.setType(LockType.SHARED_READ); break; + default: throw new MetaException("Unknown lock type " + rs.getString(6).charAt(0)); + } + e.setLastheartbeat(rs.getLong(8)); + long acquiredAt = rs.getLong(9); + if (!rs.wasNull()) e.setAcquiredat(acquiredAt); + e.setUser(rs.getString(10)); + e.setHostname(rs.getString(11)); + elems.add(e); } - switch (rs.getString(7).charAt(0)) { - case LOCK_SEMI_SHARED: e.setType(LockType.SHARED_WRITE); break; - case LOCK_EXCLUSIVE: e.setType(LockType.EXCLUSIVE); break; - case LOCK_SHARED: e.setType(LockType.SHARED_READ); break; - default: throw new MetaException("Unknown lock type " + rs.getString(6).charAt(0)); - } - e.setLastheartbeat(rs.getLong(8)); - long acquiredAt = rs.getLong(9); - if (!rs.wasNull()) e.setAcquiredat(acquiredAt); - e.setUser(rs.getString(10)); - e.setHostname(rs.getString(11)); - elems.add(e); + LOG.debug("Going to rollback"); + dbConn.rollback(); + } catch (SQLException e) { + checkRetryable(dbConn, e, "showLocks"); + throw new MetaException("Unable to select from transaction database " + + StringUtils.stringifyException(e)); + } finally { + closeStmt(stmt); + closeDbConn(dbConn); } - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e) { - throw new MetaException("Unable to select from transaction database " + - StringUtils.stringifyException(e)); - } finally { - closeStmt(stmt); - closeDbConn(dbConn); + rsp.setLocks(elems); + return rsp; + } catch (RetryException e) { + return showLocks(rqst); } - rsp.setLocks(elems); - return rsp; } public void heartbeat(HeartbeatRequest ids) - throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException { + throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); heartbeatLock(dbConn, ids.getLockid()); heartbeatTxn(dbConn, ids.getTxnid()); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "heartbeat"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "heartbeat"); throw new MetaException("Unable to select from transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { heartbeat(ids); } finally { deadlockCnt = 0; @@ -627,15 +615,16 @@ } public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst) - throws MetaException { + throws MetaException { try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; HeartbeatTxnRangeResponse rsp = new HeartbeatTxnRangeResponse(); Set nosuch = new HashSet(); Set aborted = new HashSet(); rsp.setNosuch(nosuch); rsp.setAborted(aborted); try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); for (long txn = rqst.getMin(); txn <= rqst.getMax(); txn++) { try { heartbeatTxn(dbConn, txn); @@ -647,18 +636,15 @@ } return rsp; } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "heartbeatTxnRange"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "heartbeatTxnRange"); throw new MetaException("Unable to select from transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { return heartbeatTxnRange(rqst); } } @@ -666,9 +652,10 @@ public void compact(CompactionRequest rqst) throws MetaException { // Put a compaction request in the queue. try { - Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Connection dbConn = null; Statement stmt = null; try { + dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); stmt = dbConn.createStatement(); // Get the id for the next entry in the queue @@ -679,7 +666,7 @@ LOG.debug("Going to rollback"); dbConn.rollback(); throw new MetaException("Transaction tables not properly initiated, " + - "no record found in next_compaction_queue_id"); + "no record found in next_compaction_queue_id"); } long id = rs.getLong(1); s = "update NEXT_COMPACTION_QUEUE_ID set ncq_next = " + (id + 1); @@ -687,7 +674,7 @@ stmt.executeUpdate(s); StringBuilder buf = new StringBuilder("insert into COMPACTION_QUEUE (cq_id, cq_database, " + - "cq_table, "); + "cq_table, "); String partName = rqst.getPartitionname(); if (partName != null) buf.append("cq_partition, "); buf.append("cq_state, cq_type"); @@ -730,71 +717,69 @@ LOG.debug("Going to commit"); dbConn.commit(); } catch (SQLException e) { - try { - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e1) { - } - detectDeadlock(dbConn, e, "compact"); + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "compact"); throw new MetaException("Unable to select from transaction database " + - StringUtils.stringifyException(e)); + StringUtils.stringifyException(e)); } finally { closeStmt(stmt); closeDbConn(dbConn); } - } catch (DeadlockException e) { + } catch (RetryException e) { compact(rqst); - } finally { - deadlockCnt = 0; } } public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException { ShowCompactResponse response = new ShowCompactResponse(new ArrayList()); - Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Connection dbConn = null; Statement stmt = null; try { - stmt = dbConn.createStatement(); - String s = "select cq_database, cq_table, cq_partition, cq_state, cq_type, cq_worker_id, " + + try { + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + String s = "select cq_database, cq_table, cq_partition, cq_state, cq_type, cq_worker_id, " + "cq_start, cq_run_as from COMPACTION_QUEUE"; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - while (rs.next()) { - ShowCompactResponseElement e = new ShowCompactResponseElement(); - e.setDbname(rs.getString(1)); - e.setTablename(rs.getString(2)); - e.setPartitionname(rs.getString(3)); - switch (rs.getString(4).charAt(0)) { - case INITIATED_STATE: e.setState(INITIATED_RESPONSE); break; - case WORKING_STATE: e.setState(WORKING_RESPONSE); break; - case READY_FOR_CLEANING: e.setState(CLEANING_RESPONSE); break; - default: throw new MetaException("Unexpected compaction state " + rs.getString(4)); + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + while (rs.next()) { + ShowCompactResponseElement e = new ShowCompactResponseElement(); + e.setDbname(rs.getString(1)); + e.setTablename(rs.getString(2)); + e.setPartitionname(rs.getString(3)); + switch (rs.getString(4).charAt(0)) { + case INITIATED_STATE: e.setState(INITIATED_RESPONSE); break; + case WORKING_STATE: e.setState(WORKING_RESPONSE); break; + case READY_FOR_CLEANING: e.setState(CLEANING_RESPONSE); break; + default: throw new MetaException("Unexpected compaction state " + rs.getString(4)); + } + switch (rs.getString(5).charAt(0)) { + case MAJOR_TYPE: e.setType(CompactionType.MAJOR); break; + case MINOR_TYPE: e.setType(CompactionType.MINOR); break; + default: throw new MetaException("Unexpected compaction type " + rs.getString(5)); + } + e.setWorkerid(rs.getString(6)); + e.setStart(rs.getLong(7)); + e.setRunAs(rs.getString(8)); + response.addToCompacts(e); } - switch (rs.getString(5).charAt(0)) { - case MAJOR_TYPE: e.setType(CompactionType.MAJOR); break; - case MINOR_TYPE: e.setType(CompactionType.MINOR); break; - default: throw new MetaException("Unexpected compaction type " + rs.getString(5)); - } - e.setWorkerid(rs.getString(6)); - e.setStart(rs.getLong(7)); - e.setRunAs(rs.getString(8)); - response.addToCompacts(e); - } - LOG.debug("Going to rollback"); - dbConn.rollback(); - } catch (SQLException e) { - LOG.debug("Going to rollback"); - try { + LOG.debug("Going to rollback"); dbConn.rollback(); - } catch (SQLException e1) { + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "showCompact"); + throw new MetaException("Unable to select from transaction database " + + StringUtils.stringifyException(e)); + } finally { + closeStmt(stmt); + closeDbConn(dbConn); } - throw new MetaException("Unable to select from transaction database " + - StringUtils.stringifyException(e)); - } finally { - closeStmt(stmt); - closeDbConn(dbConn); + return response; + } catch (RetryException e) { + return showCompact(rqst); } - return response; } /** @@ -828,7 +813,7 @@ return previous_timeout; } - protected class DeadlockException extends Exception { + protected class RetryException extends Exception { } @@ -839,26 +824,28 @@ * @return db connection * @throws MetaException if the connection cannot be obtained */ - protected Connection getDbConn(int isolationLevel) throws MetaException { + protected Connection getDbConn(int isolationLevel) throws SQLException { + Connection dbConn = connPool.getConnection(); + dbConn.setAutoCommit(false); + dbConn.setTransactionIsolation(isolationLevel); + return dbConn; + } + + void rollbackDBConn(Connection dbConn) { try { - Connection dbConn = connPool.getConnection(); - dbConn.setAutoCommit(false); - dbConn.setTransactionIsolation(isolationLevel); - return dbConn; + if (dbConn != null) dbConn.rollback(); } catch (SQLException e) { - String msg = "Unable to get jdbc connection from pool, " + e.getMessage(); - throw new MetaException(msg); + LOG.warn("Failed to rollback db connection " + getMessage(e)); } } - protected void closeDbConn(Connection dbConn) { try { if (dbConn != null) dbConn.close(); } catch (SQLException e) { - LOG.warn("Failed to close db connection " + e.getMessage()); + LOG.warn("Failed to close db connection " + getMessage(e)); } } - + /** * Close statement instance. * @param stmt statement instance. @@ -867,7 +854,7 @@ try { if (stmt != null) stmt.close(); } catch (SQLException e) { - LOG.warn("Failed to close statement " + e.getMessage()); + LOG.warn("Failed to close statement " + getMessage(e)); } } @@ -882,7 +869,7 @@ } } catch(SQLException ex) { - LOG.warn("Failed to close statement " + ex.getMessage()); + LOG.warn("Failed to close statement " + getMessage(ex)); } } @@ -895,18 +882,18 @@ closeDbConn(dbConn); } /** - * Determine if an exception was a deadlock. Unfortunately there is no standard way to do + * Determine if an exception was such that it makse sense to retry. Unfortunately there is no standard way to do * this, so we have to inspect the error messages and catch the telltale signs for each * different database. * @param conn database connection * @param e exception that was thrown. * @param caller name of the method calling this - * @throws org.apache.hadoop.hive.metastore.txn.TxnHandler.DeadlockException when deadlock + * @throws org.apache.hadoop.hive.metastore.txn.TxnHandler.RetryException when deadlock * detected and retry count has not been exceeded. */ - protected void detectDeadlock(Connection conn, + protected void checkRetryable(Connection conn, SQLException e, - String caller) throws DeadlockException, MetaException { + String caller) throws RetryException, MetaException { // If you change this function, remove the @Ignore from TestTxnHandler.deadlockIsDetected() // to test these changes. @@ -915,23 +902,45 @@ // so I've tried to capture the different error messages (there appear to be fewer different // error messages than SQL states). // Derby and newer MySQL driver use the new SQLTransactionRollbackException - if (dbProduct == null) { + if (dbProduct == null && conn != null) { determineDatabaseProduct(conn); } if (e instanceof SQLTransactionRollbackException || - ((dbProduct == DatabaseProduct.MYSQL || dbProduct == DatabaseProduct.POSTGRES || - dbProduct == DatabaseProduct.SQLSERVER) && e.getSQLState().equals("40001")) || - (dbProduct == DatabaseProduct.POSTGRES && e.getSQLState().equals("40P01")) || - (dbProduct == DatabaseProduct.ORACLE && (e.getMessage().contains("deadlock detected") - || e.getMessage().contains("can't serialize access for this transaction")))) { + ((dbProduct == DatabaseProduct.MYSQL || dbProduct == DatabaseProduct.POSTGRES || + dbProduct == DatabaseProduct.SQLSERVER) && e.getSQLState().equals("40001")) || + (dbProduct == DatabaseProduct.POSTGRES && e.getSQLState().equals("40P01")) || + (dbProduct == DatabaseProduct.ORACLE && (e.getMessage().contains("deadlock detected") + || e.getMessage().contains("can't serialize access for this transaction")))) { if (deadlockCnt++ < ALLOWED_REPEATED_DEADLOCKS) { LOG.warn("Deadlock detected in " + caller + ", trying again."); - throw new DeadlockException(); + throw new RetryException(); } else { LOG.error("Too many repeated deadlocks in " + caller + ", giving up."); deadlockCnt = 0; } } + else if(isRetryable(e)) { + //in MSSQL this means Communication Link Failure + if(retryNum++ < retryLimit) { + try { + Thread.sleep(retryInterval); + } + catch(InterruptedException ex) { + // + } + LOG.warn("Retryable error detected in " + caller + ", trying again: " + getMessage(e)); + throw new RetryException(); + } + else { + LOG.error("Fatal error. Retry limit (" + retryLimit + ") reached. Last error: " + getMessage(e)); + retryNum = 0; + } + } + else { + //if here, we got something that will propagate the error (rather than retry), so reset counters + deadlockCnt = 0; + retryNum = 0; + } } /** @@ -1073,10 +1082,10 @@ @Override public String toString() { return "extLockId:" + Long.toString(extLockId) + " intLockId:" + - intLockId + " txnId:" + Long.toString - (txnId) + " db:" + db + " table:" + table + " partition:" + - partition + " state:" + (state == null ? "null" : state.toString()) - + " type:" + (type == null ? "null" : type.toString()); + intLockId + " txnId:" + Long.toString + (txnId) + " db:" + db + " table:" + table + " partition:" + + partition + " state:" + (state == null ? "null" : state.toString()) + + " type:" + (type == null ? "null" : type.toString()); } } @@ -1088,11 +1097,11 @@ public int compare(LockInfo info1, LockInfo info2) { // We sort by state (acquired vs waiting) and then by extLockId. if (info1.state == LockState.ACQUIRED && - info2.state != LockState .ACQUIRED) { + info2.state != LockState .ACQUIRED) { return -1; } if (info1.state != LockState.ACQUIRED && - info2.state == LockState .ACQUIRED) { + info2.state == LockState .ACQUIRED) { return 1; } if (info1.extLockId < info2.extLockId) { @@ -1124,7 +1133,7 @@ private void checkQFileTestHack() { boolean hackOn = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST) || - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST); + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST); if (hackOn) { LOG.info("Hacking in canned values for transaction manager"); // Set up the transaction/locking db in the derby metastore @@ -1135,7 +1144,7 @@ // We may have already created the tables and thus don't need to redo it. if (!e.getMessage().contains("already exists")) { throw new RuntimeException("Unable to set up transaction database for" + - " testing: " + e.getMessage()); + " testing: " + e.getMessage()); } } } @@ -1153,7 +1162,7 @@ int updateCnt = 0; try { stmt = dbConn.createStatement(); - + // delete from HIVE_LOCKS first, we always access HIVE_LOCKS before TXNS StringBuilder buf = new StringBuilder("delete from HIVE_LOCKS where hl_txnid in ("); boolean first = true; @@ -1165,7 +1174,7 @@ buf.append(')'); LOG.debug("Going to execute update <" + buf.toString() + ">"); stmt.executeUpdate(buf.toString()); - + buf = new StringBuilder("update TXNS set txn_state = '" + TXN_ABORTED + "' where txn_id in ("); first = true; for (Long id : txnids) { @@ -1176,7 +1185,7 @@ buf.append(')'); LOG.debug("Going to execute update <" + buf.toString() + ">"); updateCnt = stmt.executeUpdate(buf.toString()); - + LOG.debug("Going to commit"); dbConn.commit(); } finally { @@ -1202,7 +1211,7 @@ * @throws TxnAbortedException */ private LockResponse lock(Connection dbConn, LockRequest rqst, boolean wait) - throws NoSuchTxnException, TxnAbortedException, MetaException, SQLException { + throws NoSuchTxnException, TxnAbortedException, MetaException, SQLException { // We want to minimize the number of concurrent lock requests being issued. If we do not we // get a large number of deadlocks in the database, since this method has to both clean // timedout locks and insert new locks. This synchronization barrier will not eliminiate all @@ -1227,7 +1236,7 @@ LOG.debug("Going to rollback"); dbConn.rollback(); throw new MetaException("Transaction tables not properly " + - "initialized, no record found in next_lock_id"); + "initialized, no record found in next_lock_id"); } long extLockId = rs.getLong(1); s = "update NEXT_LOCK_ID set nl_next = " + (extLockId + 1); @@ -1252,8 +1261,8 @@ s = "insert into TXN_COMPONENTS " + "(tc_txnid, tc_database, tc_table, tc_partition) " + "values (" + txnid + ", '" + dbName + "', " + - (tblName == null ? "null" : "'" + tblName + "'") + ", " + - (partName == null ? "null" : "'" + partName + "'") + ")"; + (tblName == null ? "null" : "'" + tblName + "'") + ", " + + (partName == null ? "null" : "'" + partName + "'") + ")"; LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); } @@ -1275,13 +1284,13 @@ long now = getDbTime(dbConn); s = "insert into HIVE_LOCKS " + " (hl_lock_ext_id, hl_lock_int_id, hl_txnid, hl_db, hl_table, " + - "hl_partition, hl_lock_state, hl_lock_type, hl_last_heartbeat, hl_user, hl_host)" + - " values (" + extLockId + ", " + - + intLockId + "," + (txnid >= 0 ? txnid : "null") + ", '" + - dbName + "', " + (tblName == null ? "null" : "'" + tblName + "'" ) - + ", " + (partName == null ? "null" : "'" + partName + "'") + - ", '" + LOCK_WAITING + "', " + "'" + lockChar + "', " + now + ", '" + - rqst.getUser() + "', '" + rqst.getHostname() + "')"; + "hl_partition, hl_lock_state, hl_lock_type, hl_last_heartbeat, hl_user, hl_host)" + + " values (" + extLockId + ", " + + + intLockId + "," + (txnid >= 0 ? txnid : "null") + ", '" + + dbName + "', " + (tblName == null ? "null" : "'" + tblName + "'" ) + + ", " + (partName == null ? "null" : "'" + partName + "'") + + ", '" + LOCK_WAITING + "', " + "'" + lockChar + "', " + now + ", '" + + rqst.getUser() + "', '" + rqst.getHostname() + "')"; LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); } @@ -1305,7 +1314,7 @@ private LockResponse checkLock(Connection dbConn, long extLockId, boolean alwaysCommit) - throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException, SQLException { + throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException, SQLException { List locksBeingChecked = getLockInfoFromLockId(dbConn, extLockId); LockResponse response = new LockResponse(); response.setLockid(extLockId); @@ -1313,8 +1322,8 @@ LOG.debug("Setting savepoint"); Savepoint save = dbConn.setSavepoint(); StringBuilder query = new StringBuilder("select hl_lock_ext_id, " + - "hl_lock_int_id, hl_db, hl_table, hl_partition, hl_lock_state, " + - "hl_lock_type from HIVE_LOCKS where hl_db in ("); + "hl_lock_int_id, hl_db, hl_table, hl_partition, hl_lock_state, " + + "hl_lock_type from HIVE_LOCKS where hl_db in ("); Set strings = new HashSet(locksBeingChecked.size()); for (LockInfo info : locksBeingChecked) { @@ -1430,7 +1439,7 @@ // lock the whole database and we need to check it. Otherwise, // check if they are operating on the same table, if not, move on. if (locks[index].table != null && locks[i].table != null - && !locks[index].table.equals(locks[i].table)) { + && !locks[index].table.equals(locks[i].table)) { continue; } @@ -1438,30 +1447,30 @@ // lock the whole table and we need to check it. Otherwise, // check if they are operating on the same partition, if not, move on. if (locks[index].partition != null && locks[i].partition != null - && !locks[index].partition.equals(locks[i].partition)) { + && !locks[index].partition.equals(locks[i].partition)) { continue; } // We've found something that matches what we're trying to lock, // so figure out if we can lock it too. switch (jumpTable.get(locks[index].type).get(locks[i].type).get - (locks[i].state)) { - case ACQUIRE: - acquire(dbConn, stmt, extLockId, info.intLockId); - acquired = true; - break; - case WAIT: - wait(dbConn, save); - if (alwaysCommit) { - // In the case where lockNoWait has been called we don't want to commit because - // it's going to roll everything back. In every other case we want to commit here. - LOG.debug("Going to commit"); - dbConn.commit(); - } - response.setState(LockState.WAITING); - return response; - case KEEP_LOOKING: - continue; + (locks[i].state)) { + case ACQUIRE: + acquire(dbConn, stmt, extLockId, info.intLockId); + acquired = true; + break; + case WAIT: + wait(dbConn, save); + if (alwaysCommit) { + // In the case where lockNoWait has been called we don't want to commit because + // it's going to roll everything back. In every other case we want to commit here. + LOG.debug("Going to commit"); + dbConn.commit(); + } + response.setState(LockState.WAITING); + return response; + case KEEP_LOOKING: + continue; } if (acquired) break; // We've acquired this lock component, // so get out of the loop and look at the next component. @@ -1494,18 +1503,18 @@ } private void acquire(Connection dbConn, Statement stmt, long extLockId, long intLockId) - throws SQLException, NoSuchLockException, MetaException { + throws SQLException, NoSuchLockException, MetaException { long now = getDbTime(dbConn); String s = "update HIVE_LOCKS set hl_lock_state = '" + LOCK_ACQUIRED + "', " + - "hl_last_heartbeat = " + now + ", hl_acquired_at = " + now + " where hl_lock_ext_id = " + - extLockId + " and hl_lock_int_id = " + intLockId; + "hl_last_heartbeat = " + now + ", hl_acquired_at = " + now + " where hl_lock_ext_id = " + + extLockId + " and hl_lock_int_id = " + intLockId; LOG.debug("Going to execute update <" + s + ">"); int rc = stmt.executeUpdate(s); if (rc < 1) { LOG.debug("Going to rollback"); dbConn.rollback(); throw new NoSuchLockException("No such lock: (" + extLockId + "," + - + intLockId + ")"); + + intLockId + ")"); } // We update the database, but we don't commit because there may be other // locks together with this, and we only want to acquire one if we can @@ -1514,7 +1523,7 @@ // Heartbeats on the lock table. This commits, so do not enter it with any state private void heartbeatLock(Connection dbConn, long extLockId) - throws NoSuchLockException, SQLException, MetaException { + throws NoSuchLockException, SQLException, MetaException { // If the lock id is 0, then there are no locks in this heartbeat if (extLockId == 0) return; Statement stmt = null; @@ -1523,7 +1532,7 @@ long now = getDbTime(dbConn); String s = "update HIVE_LOCKS set hl_last_heartbeat = " + - now + " where hl_lock_ext_id = " + extLockId; + now + " where hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute update <" + s + ">"); int rc = stmt.executeUpdate(s); if (rc < 1) { @@ -1540,7 +1549,7 @@ // Heartbeats on the txn table. This commits, so do not enter it with any state private void heartbeatTxn(Connection dbConn, long txnid) - throws NoSuchTxnException, TxnAbortedException, SQLException, MetaException { + throws NoSuchTxnException, TxnAbortedException, SQLException, MetaException { // If the txnid is 0, then there are no transactions in this heartbeat if (txnid == 0) return; Statement stmt = null; @@ -1560,10 +1569,10 @@ LOG.debug("Going to rollback"); dbConn.rollback(); throw new TxnAbortedException("Transaction " + txnid + - " already aborted"); + " already aborted"); } s = "update TXNS set txn_last_heartbeat = " + now + - " where txn_id = " + txnid; + " where txn_id = " + txnid; LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); LOG.debug("Going to commit"); @@ -1575,17 +1584,17 @@ // NEVER call this function without first calling heartbeat(long, long) private long getTxnIdFromLockId(Connection dbConn, long extLockId) - throws NoSuchLockException, MetaException, SQLException { + throws NoSuchLockException, MetaException, SQLException { Statement stmt = null; try { stmt = dbConn.createStatement(); String s = "select hl_txnid from HIVE_LOCKS where hl_lock_ext_id = " + - extLockId; + extLockId; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); if (!rs.next()) { throw new MetaException("This should never happen! We already " + - "checked the lock existed but now we can't find it!"); + "checked the lock existed but now we can't find it!"); } long txnid = rs.getLong(1); LOG.debug("Return txnid " + (rs.wasNull() ? -1 : txnid)); @@ -1597,13 +1606,13 @@ // NEVER call this function without first calling heartbeat(long, long) private List getLockInfoFromLockId(Connection dbConn, long extLockId) - throws NoSuchLockException, MetaException, SQLException { + throws NoSuchLockException, MetaException, SQLException { Statement stmt = null; try { stmt = dbConn.createStatement(); String s = "select hl_lock_ext_id, hl_lock_int_id, hl_db, hl_table, " + - "hl_partition, hl_lock_state, hl_lock_type from HIVE_LOCKS where " + - "hl_lock_ext_id = " + extLockId; + "hl_partition, hl_lock_state, hl_lock_type from HIVE_LOCKS where " + + "hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); boolean sawAtLeastOne = false; @@ -1614,7 +1623,7 @@ } if (!sawAtLeastOne) { throw new MetaException("This should never happen! We already " + - "checked the lock existed but now we can't find it!"); + "checked the lock existed but now we can't find it!"); } return ourLockInfo; } finally { @@ -1632,7 +1641,7 @@ stmt = dbConn.createStatement(); // Remove any timed out locks from the table. String s = "delete from HIVE_LOCKS where hl_last_heartbeat < " + - (now - timeout); + (now - timeout); LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); LOG.debug("Going to commit"); @@ -1652,7 +1661,7 @@ stmt = dbConn.createStatement(); // Abort any timed out locks from the table. String s = "select txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); + "' and txn_last_heartbeat < " + (now - timeout); LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); List deadTxns = new ArrayList(); @@ -1675,12 +1684,12 @@ String passwd; try { passwd = ShimLoader.getHadoopShims().getPassword(conf, - HiveConf.ConfVars.METASTOREPWD.varname); + HiveConf.ConfVars.METASTOREPWD.varname); } catch (IOException err) { throw new SQLException("Error getting metastore password", err); } String connectionPooler = HiveConf.getVar(conf, - HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); + HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); if ("bonecp".equals(connectionPooler)) { BoneCPConfig config = new BoneCPConfig(); @@ -1696,22 +1705,22 @@ // This doesn't get used, but it's still necessary, see // http://svn.apache.org/viewvc/commons/proper/dbcp/branches/DBCP_1_4_x_BRANCH/doc/ManualPoolingDataSourceExample.java?view=markup PoolableConnectionFactory poolConnFactory = - new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true); + new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true); connPool = new PoolingDataSource(objectPool); } else { throw new RuntimeException("Unknown JDBC connection pooling " + connectionPooler); } } - private static synchronized void buildJumpTable() { + private static synchronized void buildJumpTable() { if (jumpTable != null) return; jumpTable = - new HashMap>>(3); + new HashMap>>(3); // SR: Lock we are trying to acquire is shared read Map> m = - new HashMap>(3); + new HashMap>(3); jumpTable.put(LockType.SHARED_READ, m); // SR.SR: Lock we are examining is shared read @@ -1743,7 +1752,7 @@ // that something is blocking it that would not block a read. m2.put(LockState.WAITING, LockAction.KEEP_LOOKING); - // SR.E: Lock we are examining is exclusive + // SR.E: Lock we are examining is exclusive m2 = new HashMap(2); m.put(LockType.EXCLUSIVE, m2); @@ -1777,7 +1786,7 @@ m2.put(LockState.ACQUIRED, LockAction.WAIT); m2.put(LockState.WAITING, LockAction.WAIT); - // SW.E: Lock we are examining is exclusive + // SW.E: Lock we are examining is exclusive m2 = new HashMap(2); m.put(LockType.EXCLUSIVE, m2); @@ -1805,7 +1814,7 @@ m2.put(LockState.ACQUIRED, LockAction.WAIT); m2.put(LockState.WAITING, LockAction.WAIT); - // E.E: Lock we are examining is exclusive + // E.E: Lock we are examining is exclusive m2 = new HashMap(2); m.put(LockType.EXCLUSIVE, m2); @@ -1813,4 +1822,20 @@ m2.put(LockState.ACQUIRED, LockAction.WAIT); m2.put(LockState.WAITING, LockAction.WAIT); } + /** + * Returns true if {@code ex} should be retried + */ + private static boolean isRetryable(Exception ex) { + if(ex instanceof SQLException) { + SQLException sqlException = (SQLException)ex; + if("08S01".equalsIgnoreCase(sqlException.getSQLState())) { + //in MSSQL this means Communication Link Failure + return true; + } + } + return false; + } + private static String getMessage(SQLException ex) { + return ex.getMessage() + "(SQLState=" + ex.getSQLState() + ",ErrorCode=" + ex.getErrorCode() + ")"; + } } Index: metastore/src/model/package.jdo =================================================================== --- metastore/src/model/package.jdo (revision 1653459) +++ metastore/src/model/package.jdo (working copy) @@ -1005,7 +1005,7 @@ - + Index: metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java (revision 1653459) +++ metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java (working copy) @@ -1124,11 +1124,11 @@ LOG.debug("no exception, no deadlock"); } catch (SQLException e) { try { - txnHandler.detectDeadlock(conn1, e, "thread t1"); + txnHandler.checkRetryable(conn1, e, "thread t1"); LOG.debug("Got an exception, but not a deadlock, SQLState is " + e.getSQLState() + " class of exception is " + e.getClass().getName() + " msg is <" + e.getMessage() + ">"); - } catch (TxnHandler.DeadlockException de) { + } catch (TxnHandler.RetryException de) { LOG.debug("Forced a deadlock, SQLState is " + e.getSQLState() + " class of " + "exception is " + e.getClass().getName() + " msg is <" + e .getMessage() + ">"); @@ -1154,11 +1154,11 @@ LOG.debug("no exception, no deadlock"); } catch (SQLException e) { try { - txnHandler.detectDeadlock(conn2, e, "thread t2"); + txnHandler.checkRetryable(conn2, e, "thread t2"); LOG.debug("Got an exception, but not a deadlock, SQLState is " + e.getSQLState() + " class of exception is " + e.getClass().getName() + " msg is <" + e.getMessage() + ">"); - } catch (TxnHandler.DeadlockException de) { + } catch (TxnHandler.RetryException de) { LOG.debug("Forced a deadlock, SQLState is " + e.getSQLState() + " class of " + "exception is " + e.getClass().getName() + " msg is <" + e .getMessage() + ">"); Index: metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java (revision 0) +++ metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java (working copy) @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.junit.Test; + +public class TestTxnHandlerNegative { + static final private Log LOG = LogFactory.getLog(TestTxnHandlerNegative.class); + + /** + * this intentionally sets a bad URL for connection to test error handling logic + * in TxnHandler + * @throws Exception + */ + @Test + public void testBadConnection() throws Exception { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "blah"); + TxnHandler txnHandler1 = new TxnHandler(conf); + MetaException e = null; + try { + txnHandler1.getOpenTxns(); + } + catch(MetaException ex) { + LOG.info("Expected error: " + ex.getMessage(), ex); + e = ex; + } + assert e != null : "did not get exception"; + } +} Index: pom.xml =================================================================== --- pom.xml (revision 1653459) +++ pom.xml (working copy) @@ -119,7 +119,7 @@ 14.0.1 2.1.6 1.2.1 - 2.5.0 + 2.6.0 ${basedir}/${hive.path.to.root}/testutils/hadoop 0.98.3-hadoop1 0.98.3-hadoop2 @@ -351,9 +351,9 @@ ${log4j.version} - log4j - apache-log4j-extras - ${log4j-extras.version} + log4j + apache-log4j-extras + ${log4j-extras.version} org.antlr Index: ql/pom.xml =================================================================== --- ql/pom.xml (revision 1653459) +++ ql/pom.xml (working copy) @@ -173,6 +173,23 @@ ${zookeeper.version} + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + apache-curator + ${curator.version} + pom + + + org.apache.curator + curator-test + ${curator.version} + test + + org.codehaus.groovy groovy-all ${groovy.version} Index: ql/src/java/org/apache/hadoop/hive/ql/Context.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Context.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java (working copy) @@ -90,7 +90,7 @@ protected int tryCount = 0; private TokenRewriteStream tokenRewriteStream; - String executionId; + private String executionId; // List of Locks for this query protected List hiveLocks; @@ -112,6 +112,8 @@ private final Map> outputLockObjects = new HashMap>(); + private final String stagingDir; + public Context(Configuration conf) throws IOException { this(conf, generateExecutionId()); } @@ -129,6 +131,7 @@ nonLocalScratchPath = new Path(SessionState.getHDFSSessionPath(conf), executionId); localScratchDir = new Path(SessionState.getLocalSessionPath(conf), executionId).toUri().getPath(); scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION); + stagingDir = HiveConf.getVar(conf, HiveConf.ConfVars.STAGINGDIR); } @@ -188,6 +191,65 @@ } /** + * Gets a temporary staging directory related to a path. + * If a path already contains a staging directory, then returns the current directory; otherwise + * create the directory if needed. + * + * @param inputPath URI of the temporary directory + * @param mkdir Create the directory if True. + * @return A temporary path. + */ + private Path getStagingDir(Path inputPath, boolean mkdir) { + final URI inputPathUri = inputPath.toUri(); + final String inputPathName = inputPathUri.getPath(); + final String fileSystem = inputPathUri.getScheme() + ":" + inputPathUri.getAuthority(); + final FileSystem fs; + + try { + fs = inputPath.getFileSystem(conf); + } catch (IOException e) { + throw new IllegalStateException("Error getting FileSystem for " + inputPath + ": "+ e, e); + } + + String stagingPathName; + if (inputPathName.indexOf(stagingDir) == -1) { + stagingPathName = new Path(inputPathName, stagingDir).toString(); + } else { + stagingPathName = inputPathName.substring(0, inputPathName.indexOf(stagingDir) + stagingDir.length()); + } + + final String key = fileSystem + "-" + stagingPathName + "-" + TaskRunner.getTaskRunnerID(); + + Path dir = fsScratchDirs.get(key); + if (dir == null) { + // Append task specific info to stagingPathName, instead of creating a sub-directory. + // This way we don't have to worry about deleting the stagingPathName separately at + // end of query execution. + dir = fs.makeQualified(new Path(stagingPathName + "_" + this.executionId + "-" + TaskRunner.getTaskRunnerID())); + + LOG.debug("Created staging dir = " + dir + " for path = " + inputPath); + + if (mkdir) { + try { + if (!FileUtils.mkdir(fs, dir, true, conf)) { + throw new IllegalStateException("Cannot create staging directory '" + dir.toString() + "'"); + } + + if (isHDFSCleanup) { + fs.deleteOnExit(dir); + } + } catch (IOException e) { + throw new RuntimeException("Cannot create staging directory '" + dir.toString() + "': " + e.getMessage(), e); + } + } + + fsScratchDirs.put(key, dir); + } + + return dir; + } + + /** * Get a tmp directory on specified URI * * @param scheme Scheme of the target FS @@ -274,14 +336,13 @@ } private Path getExternalScratchDir(URI extURI) { - return getScratchDir(extURI.getScheme(), extURI.getAuthority(), - !explain, nonLocalScratchPath.toUri().getPath()); + return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain); } /** * Remove any created scratch directories. */ - private void removeScratchDir() { + public void removeScratchDir() { for (Map.Entry entry : fsScratchDirs.entrySet()) { try { Path p = entry.getValue(); @@ -313,6 +374,10 @@ (uriStr.indexOf(MR_PREFIX) != -1); } + public Path getMRTmpPath(URI uri) { + return new Path(getStagingDir(new Path(uri), !explain), MR_PREFIX + nextPathId()); + } + /** * Get a path to store map-reduce intermediate data in. * @@ -333,10 +398,9 @@ } /** - * Get a path to store tmp data destined for external URI. + * Get a path to store tmp data destined for external Path. * - * @param extURI - * external URI to which the tmp data has to be eventually moved + * @param path external Path to which the tmp data has to be eventually moved * @return next available tmp path on the file system corresponding extURI */ public Path getExternalTmpPath(Path path) { @@ -357,9 +421,7 @@ * path within /tmp */ public Path getExtTmpPathRelTo(Path path) { - URI uri = path.toUri(); - return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, - uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId()); + return new Path(getStagingDir(path, !explain), EXT_PREFIX + nextPathId()); } /** @@ -437,7 +499,7 @@ resFs = resDir.getFileSystem(conf); FileStatus status = resFs.getFileStatus(resDir); assert status.isDir(); - FileStatus[] resDirFS = resFs.globStatus(new Path(resDir + "/*")); + FileStatus[] resDirFS = resFs.globStatus(new Path(resDir + "/*"), FileUtils.HIDDEN_FILES_PATH_FILTER); resDirPaths = new Path[resDirFS.length]; int pos = 0; for (FileStatus resFS : resDirFS) { Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.hooks.PreExecute; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.hooks.Redactor; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; @@ -439,6 +440,11 @@ SessionState.get().getCommandType()); String queryStr = plan.getQueryStr(); + List queryRedactors = getHooks(ConfVars.QUERYREDACTORHOOKS, Redactor.class); + for (Redactor redactor : queryRedactors) { + redactor.setConf(conf); + queryStr = redactor.redactQuery(queryStr); + } conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr); conf.set("mapreduce.workflow.id", "hive_" + queryId); @@ -699,15 +705,13 @@ || op.equals(HiveOperation.QUERY)) { SemanticAnalyzer querySem = (SemanticAnalyzer) sem; ParseContext parseCtx = querySem.getParseContext(); - Map tsoTopMap = parseCtx.getTopToTable(); for (Map.Entry> topOpMap : querySem .getParseContext().getTopOps().entrySet()) { Operator topOp = topOpMap.getValue(); - if (topOp instanceof TableScanOperator - && tsoTopMap.containsKey(topOp)) { + if (topOp instanceof TableScanOperator) { TableScanOperator tableScanOp = (TableScanOperator) topOp; - Table tbl = tsoTopMap.get(tableScanOp); + Table tbl = tableScanOp.getConf().getTableMetadata(); List neededColumnIds = tableScanOp.getNeededColumnIDs(); List columns = tbl.getCols(); List cols = new ArrayList(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (working copy) @@ -64,6 +64,20 @@ "Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc); } + public static ExprNodeEvaluator[] toCachedEvals(ExprNodeEvaluator[] evals) { + EvaluatorContext context = new EvaluatorContext(); + for (int i = 0; i < evals.length; i++) { + if (evals[i] instanceof ExprNodeGenericFuncEvaluator) { + iterate(evals[i], context); + if (context.hasReference) { + evals[i] = new ExprNodeEvaluatorHead(evals[i]); + context.hasReference = false; + } + } + } + return evals; + } + /** * Should be called before eval is initialized */ Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (working copy) @@ -637,10 +637,10 @@ boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE); // If this is in acid format always read it recursively regardless of what the jobconf says. if (!recursive && !AcidUtils.isAcid(p, job)) { - return fs.listStatus(p); + return fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER); } List results = new ArrayList(); - for (FileStatus stat : fs.listStatus(p)) { + for (FileStatus stat : fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER)) { FileUtils.listStatusRecursively(fs, stat, results); } return results.toArray(new FileStatus[results.size()]); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (working copy) @@ -68,11 +68,16 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyShim; +import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyValue; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.ReflectionUtils; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE_STORAGE; + /** * File Sink operator implementation. **/ @@ -88,6 +93,7 @@ protected transient List dpColNames; protected transient DynamicPartitionCtx dpCtx; protected transient boolean isCompressed; + protected transient boolean isTemporary; protected transient Path parent; protected transient HiveOutputFormat hiveOutputFormat; protected transient Path specPath; @@ -318,6 +324,7 @@ this.hconf = hconf; filesCreated = false; isNativeTable = !conf.getTableInfo().isNonNative(); + isTemporary = conf.isTemporary(); multiFileSpray = conf.isMultiFileSpray(); totalFiles = conf.getTotalFiles(); numFiles = conf.getNumFiles(); @@ -384,6 +391,20 @@ valToPaths.put("", fsp); // special entry for non-DP case } } + + final StoragePolicyValue tmpStorage = StoragePolicyValue.lookup(HiveConf + .getVar(hconf, HIVE_TEMPORARY_TABLE_STORAGE)); + if (isTemporary && fsp != null + && tmpStorage != StoragePolicyValue.DEFAULT) { + final Path outputPath = fsp.taskOutputTempPath; + StoragePolicyShim shim = ShimLoader.getHadoopShims() + .getStoragePolicyShim(fs); + if (shim != null) { + // directory creation is otherwise within the writers + fs.mkdirs(outputPath); + shim.setStoragePolicy(outputPath, tmpStorage); + } + } if (conf.getWriteType() == AcidUtils.Operation.UPDATE || conf.getWriteType() == AcidUtils.Operation.DELETE) { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -41,9 +41,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -248,6 +246,7 @@ registerUDF("reverse", UDFReverse.class, false); registerGenericUDF("field", GenericUDFField.class); registerUDF("find_in_set", UDFFindInSet.class, false); + registerGenericUDF("initcap", GenericUDFInitCap.class); registerUDF("like", UDFLike.class, true); registerUDF("rlike", UDFRegExp.class, true); @@ -273,10 +272,12 @@ registerUDF("from_unixtime", UDFFromUnixTime.class, false); registerGenericUDF("to_date", GenericUDFDate.class); registerUDF("weekofyear", UDFWeekOfYear.class, false); + registerGenericUDF("last_day", GenericUDFLastDay.class); registerGenericUDF("date_add", GenericUDFDateAdd.class); registerGenericUDF("date_sub", GenericUDFDateSub.class); registerGenericUDF("datediff", GenericUDFDateDiff.class); + registerGenericUDF("add_months", GenericUDFAddMonths.class); registerUDF("get_json_object", UDFJson.class, false); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (working copy) @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.Serializable; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.reflect.Field; @@ -34,15 +33,12 @@ import javolution.util.FastBitSet; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -72,115 +68,110 @@ /** * GroupBy operator implementation. */ -public class GroupByOperator extends Operator implements - Serializable { +public class GroupByOperator extends Operator { - private static final Log LOG = LogFactory.getLog(GroupByOperator.class - .getName()); - private static final boolean isTraceEnabled = LOG.isTraceEnabled(); private static final long serialVersionUID = 1L; private static final int NUMROWSESTIMATESIZE = 1000; - protected transient ExprNodeEvaluator[] keyFields; - protected transient ObjectInspector[] keyObjectInspectors; + private transient ExprNodeEvaluator[] keyFields; + private transient ObjectInspector[] keyObjectInspectors; - protected transient ExprNodeEvaluator[][] aggregationParameterFields; - protected transient ObjectInspector[][] aggregationParameterObjectInspectors; - protected transient ObjectInspector[][] aggregationParameterStandardObjectInspectors; - protected transient Object[][] aggregationParameterObjects; + private transient ExprNodeEvaluator[][] aggregationParameterFields; + private transient ObjectInspector[][] aggregationParameterObjectInspectors; + private transient ObjectInspector[][] aggregationParameterStandardObjectInspectors; + private transient Object[][] aggregationParameterObjects; + // so aggregationIsDistinct is a boolean array instead of a single number. - protected transient boolean[] aggregationIsDistinct; + private transient boolean[] aggregationIsDistinct; // Map from integer tag to distinct aggrs - transient protected Map> distinctKeyAggrs = + private transient Map> distinctKeyAggrs = new HashMap>(); // Map from integer tag to non-distinct aggrs with key parameters. - transient protected Map> nonDistinctKeyAggrs = + private transient Map> nonDistinctKeyAggrs = new HashMap>(); // List of non-distinct aggrs. - transient protected List nonDistinctAggrs = new ArrayList(); + private transient List nonDistinctAggrs = new ArrayList(); // Union expr for distinct keys - transient ExprNodeEvaluator unionExprEval = null; + private transient ExprNodeEvaluator unionExprEval; - transient GenericUDAFEvaluator[] aggregationEvaluators; - transient boolean[] estimableAggregationEvaluators; + private transient GenericUDAFEvaluator[] aggregationEvaluators; + private transient boolean[] estimableAggregationEvaluators; - protected transient ArrayList objectInspectors; - transient ArrayList fieldNames; - // Used by sort-based GroupBy: Mode = COMPLETE, PARTIAL1, PARTIAL2, // MERGEPARTIAL - protected transient KeyWrapper currentKeys; - protected transient KeyWrapper newKeys; - protected transient AggregationBuffer[] aggregations; - protected transient Object[][] aggregationsParametersLastInvoke; + private transient KeyWrapper currentKeys; + private transient KeyWrapper newKeys; + private transient AggregationBuffer[] aggregations; + private transient Object[][] aggregationsParametersLastInvoke; // Used by hash-based GroupBy: Mode = HASH, PARTIALS - protected transient HashMap hashAggregations; + private transient HashMap hashAggregations; // Used by hash distinct aggregations when hashGrpKeyNotRedKey is true - protected transient HashSet keysCurrentGroup; + private transient HashSet keysCurrentGroup; - transient boolean firstRow; - transient long totalMemory; - protected transient boolean hashAggr; + private transient boolean firstRow; + private transient boolean hashAggr; // The reduction is happening on the reducer, and the grouping key and // reduction keys are different. // For example: select a, count(distinct b) from T group by a // The data is sprayed by 'b' and the reducer is grouping it by 'a' - transient boolean groupKeyIsNotReduceKey; - transient boolean firstRowInGroup; - transient long numRowsInput; - transient long numRowsHashTbl; - transient int groupbyMapAggrInterval; - transient long numRowsCompareHashAggr; - transient float minReductionHashAggr; + private transient boolean groupKeyIsNotReduceKey; + private transient boolean firstRowInGroup; + private transient long numRowsInput; + private transient long numRowsHashTbl; + private transient int groupbyMapAggrInterval; + private transient long numRowsCompareHashAggr; + private transient float minReductionHashAggr; + private transient int outputKeyLength; + // current Key ObjectInspectors are standard ObjectInspectors - protected transient ObjectInspector[] currentKeyObjectInspectors; - // new Key ObjectInspectors are objectInspectors from the parent - transient StructObjectInspector newKeyObjectInspector; - transient StructObjectInspector currentKeyObjectInspector; - public static MemoryMXBean memoryMXBean; + private transient ObjectInspector[] currentKeyObjectInspectors; - /** - * Total amount of memory allowed for JVM heap. - */ - protected long maxMemory; + private transient MemoryMXBean memoryMXBean; - /** - * configure percent of memory threshold usable by QP. - */ - protected float memoryThreshold; + private transient boolean groupingSetsPresent; // generates grouping set + private transient int groupingSetsPosition; // position of grouping set, generally the last of keys + private transient List groupingSets; // declared grouping set values + private transient FastBitSet[] groupingSetsBitSet; // bitsets acquired from grouping set values + private transient Text[] newKeysGroupingSets; - private boolean groupingSetsPresent; - private int groupingSetsPosition; - private List groupingSets; - private List groupingSetsBitSet; - transient private List newKeysGroupingSets; - // for these positions, some variable primitive type (String) is used, so size // cannot be estimated. sample it at runtime. - transient List keyPositionsSize; + private transient List keyPositionsSize; // for these positions, some variable primitive type (String) is used for the // aggregation classes - transient List[] aggrPositions; + private transient List[] aggrPositions; - transient int fixedRowSize; + private transient int fixedRowSize; + private transient int totalVariableSize; + private transient int numEntriesVarSize; + + private transient int countAfterReport; // report or forward + private transient int heartbeatInterval; + /** + * Total amount of memory allowed for JVM heap. + */ + protected transient long maxMemory; + + /** * Max memory usable by the hashtable before it should flush. */ protected transient long maxHashTblMemory; - transient int totalVariableSize; - transient int numEntriesVarSize; /** + * configure percent of memory threshold usable by QP. + */ + protected transient float memoryThreshold; + + /** * Current number of entries in the hash table. */ protected transient int numEntriesHashTable; - transient int countAfterReport; // report or forward - transient int heartbeatInterval; public static FastBitSet groupingSet2BitSet(int value) { FastBitSet bits = new FastBitSet(); @@ -197,7 +188,6 @@ @Override protected void initializeOp(Configuration hconf) throws HiveException { - totalMemory = Runtime.getRuntime().totalMemory(); numRowsInput = 0; numRowsHashTbl = 0; @@ -226,16 +216,15 @@ if (groupingSetsPresent) { groupingSets = conf.getListGroupingSets(); groupingSetsPosition = conf.getGroupingSetPosition(); - newKeysGroupingSets = new ArrayList(); - groupingSetsBitSet = new ArrayList(); + newKeysGroupingSets = new Text[groupingSets.size()]; + groupingSetsBitSet = new FastBitSet[groupingSets.size()]; + int pos = 0; for (Integer groupingSet: groupingSets) { // Create the mapping corresponding to the grouping set - ExprNodeEvaluator groupingSetValueEvaluator = - ExprNodeEvaluatorFactory.get(new ExprNodeConstantDesc(String.valueOf(groupingSet))); - - newKeysGroupingSets.add(groupingSetValueEvaluator.evaluate(null)); - groupingSetsBitSet.add(groupingSet2BitSet(groupingSet)); + newKeysGroupingSets[pos] = new Text(String.valueOf(groupingSet)); + groupingSetsBitSet[pos] = groupingSet2BitSet(groupingSet); + pos++; } } @@ -348,23 +337,12 @@ aggregationEvaluators[i] = agg.getGenericUDAFEvaluator(); } - // init objectInspectors - int totalFields = keyFields.length + aggregationEvaluators.length; - objectInspectors = new ArrayList(totalFields); - for (ExprNodeEvaluator keyField : keyFields) { - objectInspectors.add(null); - } MapredContext context = MapredContext.get(); if (context != null) { for (GenericUDAFEvaluator genericUDAFEvaluator : aggregationEvaluators) { context.setup(genericUDAFEvaluator); } } - for (int i = 0; i < aggregationEvaluators.length; i++) { - ObjectInspector roi = aggregationEvaluators[i].init(conf.getAggregators() - .get(i).getMode(), aggregationParameterObjectInspectors[i]); - objectInspectors.add(roi); - } aggregationsParametersLastInvoke = new Object[conf.getAggregators().size()][]; if ((conf.getMode() != GroupByDesc.Mode.HASH || conf.getBucketGroup()) && @@ -390,26 +368,25 @@ } } - fieldNames = conf.getOutputColumnNames(); + List fieldNames = new ArrayList(conf.getOutputColumnNames()); - for (int i = 0; i < keyFields.length; i++) { - objectInspectors.set(i, currentKeyObjectInspectors[i]); - } + // grouping id should be pruned, which is the last of key columns + // see ColumnPrunerGroupByProc + outputKeyLength = conf.pruneGroupingSetId() ? keyFields.length - 1 : keyFields.length; - // Generate key names - ArrayList keyNames = new ArrayList(keyFields.length); - for (int i = 0; i < keyFields.length; i++) { - keyNames.add(fieldNames.get(i)); + // init objectInspectors + ObjectInspector[] objectInspectors = + new ObjectInspector[outputKeyLength + aggregationEvaluators.length]; + for (int i = 0; i < outputKeyLength; i++) { + objectInspectors[i] = currentKeyObjectInspectors[i]; } - newKeyObjectInspector = ObjectInspectorFactory - .getStandardStructObjectInspector(keyNames, Arrays - .asList(keyObjectInspectors)); - currentKeyObjectInspector = ObjectInspectorFactory - .getStandardStructObjectInspector(keyNames, Arrays - .asList(currentKeyObjectInspectors)); + for (int i = 0; i < aggregationEvaluators.length; i++) { + objectInspectors[outputKeyLength + i] = aggregationEvaluators[i].init(conf.getAggregators() + .get(i).getMode(), aggregationParameterObjectInspectors[i]); + } outputObjInspector = ObjectInspectorFactory - .getStandardStructObjectInspector(fieldNames, objectInspectors); + .getStandardStructObjectInspector(fieldNames, Arrays.asList(objectInspectors)); KeyWrapperFactory keyWrapperFactory = new KeyWrapperFactory(keyFields, keyObjectInspectors, currentKeyObjectInspectors); @@ -769,7 +746,7 @@ flushHashTable(true); hashAggr = false; } else { - if (isTraceEnabled) { + if (isLogTraceEnabled) { LOG.trace("Hash Aggr Enabled: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput + " reduction = " + 1.0 * (numRowsHashTbl / numRowsInput) + " minReduction = " @@ -795,14 +772,14 @@ newKeysArray[keyPos] = null; } - FastBitSet bitset = groupingSetsBitSet.get(groupingSetPos); + FastBitSet bitset = groupingSetsBitSet[groupingSetPos]; // Some keys need to be left to null corresponding to that grouping set. for (int keyPos = bitset.nextSetBit(0); keyPos >= 0; keyPos = bitset.nextSetBit(keyPos+1)) { newKeysArray[keyPos] = cloneNewKeysArray[keyPos]; } - newKeysArray[groupingSetsPosition] = newKeysGroupingSets.get(groupingSetPos); + newKeysArray[groupingSetsPosition] = newKeysGroupingSets[groupingSetPos]; processKey(row, rowInspector); } } else { @@ -972,7 +949,7 @@ // Update the number of entries that can fit in the hash table numEntriesHashTable = (int) (maxHashTblMemory / (fixedRowSize + (totalVariableSize / numEntriesVarSize))); - if (isTraceEnabled) { + if (isLogTraceEnabled) { LOG.trace("Hash Aggr: #hash table = " + numEntries + " #max in hash table = " + numEntriesHashTable); } @@ -1054,19 +1031,17 @@ * The keys in the record * @throws HiveException */ - protected void forward(Object[] keys, - AggregationBuffer[] aggs) throws HiveException { + private void forward(Object[] keys, AggregationBuffer[] aggs) throws HiveException { - int totalFields = keys.length + aggs.length; if (forwardCache == null) { - forwardCache = new Object[totalFields]; + forwardCache = new Object[outputKeyLength + aggs.length]; } - for (int i = 0; i < keys.length; i++) { + for (int i = 0; i < outputKeyLength; i++) { forwardCache[i] = keys[i]; } for (int i = 0; i < aggs.length; i++) { - forwardCache[keys.length + i] = aggregationEvaluators[i].evaluate(aggs[i]); + forwardCache[outputKeyLength + i] = aggregationEvaluators[i].evaluate(aggs[i]); } forward(forwardCache, outputObjInspector); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (working copy) @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -58,6 +59,8 @@ import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.util.StringUtils; import java.io.IOException; @@ -99,7 +102,7 @@ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INSERT_INTO_MULTILEVEL_DIRS)) { deletePath = createTargetPath(targetPath, fs); } - if (!Hive.renameFile(conf, sourcePath, targetPath, fs, true, false)) { + if (!Hive.moveFile(conf, sourcePath, targetPath, fs, true, false)) { try { if (deletePath != null) { fs.delete(deletePath, true); @@ -158,8 +161,14 @@ actualPath = actualPath.getParent(); } fs.mkdirs(mkDirPath); + HadoopShims shims = ShimLoader.getHadoopShims(); if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS)) { - fs.setPermission(mkDirPath, fs.getFileStatus(actualPath).getPermission()); + try { + HadoopShims.HdfsFileStatus status = shims.getFullFileStatus(conf, fs, actualPath); + shims.setFullFileStatus(conf, status, fs, actualPath); + } catch (Exception e) { + LOG.warn("Error setting permissions or group of " + actualPath, e); + } } } return deletePath; @@ -259,7 +268,7 @@ dirs = srcFs.globStatus(tbd.getSourcePath()); files = new ArrayList(); for (int i = 0; (dirs != null && i < dirs.length); i++) { - files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath()))); + files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER))); // We only check one file, so exit the loop when we have at least // one. if (files.size() > 0) { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (working copy) @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; /** * Select operator implementation. @@ -55,12 +54,12 @@ for (int i = 0; i < colList.size(); i++) { assert (colList.get(i) != null); eval[i] = ExprNodeEvaluatorFactory.get(colList.get(i)); - if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) { - eval[i] = ExprNodeEvaluatorFactory.toCachedEval(eval[i]); - } } + if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) { + eval = ExprNodeEvaluatorFactory.toCachedEvals(eval); + } output = new Object[eval.length]; - LOG.info("SELECT " + ((StructObjectInspector) inputObjInspectors[0]).getTypeName()); + LOG.info("SELECT " + inputObjInspectors[0].getTypeName()); outputObjInspector = initEvaluatorsAndReturnStruct(eval, conf.getOutputColumnNames(), inputObjInspectors[0]); initializeChildren(hconf); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -93,6 +93,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveInterruptCallback; import org.apache.hadoop.hive.common.HiveInterruptUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; @@ -1068,7 +1069,6 @@ removeField(kryo, Operator.class, "colExprMap"); removeField(kryo, ColumnInfo.class, "objectInspector"); removeField(kryo, MapWork.class, "opParseCtxMap"); - removeField(kryo, MapWork.class, "joinTree"); return kryo; }; }; @@ -1803,7 +1803,7 @@ */ public static FileStatus[] listStatusIfExists(Path path, FileSystem fs) throws IOException { try { - return fs.listStatus(path); + return fs.listStatus(path, FileUtils.HIDDEN_FILES_PATH_FILTER); } catch (FileNotFoundException e) { // FS in hadoop 2.0 throws FNF instead of returning null return null; @@ -2639,7 +2639,7 @@ FileSystem inpFs = dirPath.getFileSystem(job); if (inpFs.exists(dirPath)) { - FileStatus[] fStats = inpFs.listStatus(dirPath); + FileStatus[] fStats = inpFs.listStatus(dirPath, FileUtils.HIDDEN_FILES_PATH_FILTER); if (fStats.length > 0) { return false; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java (working copy) @@ -0,0 +1,401 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import java.io.ObjectOutputStream; +import java.util.AbstractCollection; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.NoSuchElementException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Writable; + +@SuppressWarnings("deprecation") +public class FlatRowContainer extends AbstractCollection + implements MapJoinRowContainer, AbstractRowContainer.RowIterator>, List { + private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + private static final int UNKNOWN = Integer.MAX_VALUE; + + private static Log LOG = LogFactory.getLog(FlatRowContainer.class); + + /** + * In lazy mode, 0s element contains context for deserialization and all the other + * elements contains byte arrays to be deserialized. After deserialization, the array + * contains row count * row size elements - a matrix of rows stored. + */ + private Object[] array; + /** + * This is kind of tricky. UNKNOWN number means unknown. Other positive numbers represent + * row length (see array javadoc). Non-positive numbers mean row length is zero (thus, + * array is empty); they represent (negated) number of rows (for joins w/o projections). + */ + private int rowLength = UNKNOWN; + private byte aliasFilter = (byte) 0xff; + private boolean isAliasFilterSet = true; // by default assume no filter tag so we are good + + public FlatRowContainer() { + this.array = EMPTY_OBJECT_ARRAY; + } + + /** Called when loading the hashtable. */ + public void add(MapJoinObjectSerDeContext context, + BytesWritable value) throws HiveException { + SerDe serde = context.getSerDe(); + isAliasFilterSet = !context.hasFilterTag(); // has tag => need to set later + if (rowLength == UNKNOWN) { + try { + rowLength = ObjectInspectorUtils.getStructSize(serde.getObjectInspector()); + } catch (SerDeException ex) { + throw new HiveException("Get structure size error", ex); + } + if (rowLength == 0) { + array = EMPTY_OBJECT_ARRAY; + } + } + if (rowLength > 0) { + int rowCount = (array.length / rowLength); + listRealloc(array.length + rowLength); + read(serde, value, rowCount); + } else { + --rowLength; // see rowLength javadoc + } + } + + // Implementation of AbstractRowContainer and assorted methods + + @Override + public void addRow(List t) throws HiveException { + LOG.debug("Add is called with " + t.size() + " objects"); + // This is not called when building HashTable; we don't expect it to be called ever. + int offset = prepareForAdd(t.size()); + if (offset < 0) return; + for (int i = 0; i < t.size(); ++i) { + this.array[offset + i] = t.get(i); + } + } + + @Override + public void addRow(Object[] value) throws HiveException { + LOG.debug("Add is called with " + value.length + " objects"); + // This is not called when building HashTable; we don't expect it to be called ever. + int offset = prepareForAdd(value.length); + if (offset < 0) return; + System.arraycopy(value, 0, this.array, offset, value.length); + } + + private int prepareForAdd(int len) throws HiveException { + if (rowLength < 0) { + if (len != 0) { + throw new HiveException("Different size rows: 0 and " + len); + } + --rowLength; // see rowLength javadoc + return -1; + } + if (rowLength != len) { + throw new HiveException("Different size rows: " + rowLength + " and " + len); + } + int oldLen = this.array.length; + listRealloc(oldLen + len); + return oldLen; + } + + @Override + public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out) { + throw new UnsupportedOperationException(this.getClass().getName() + " cannot be serialized"); + } + + @Override + public AbstractRowContainer.RowIterator> rowIter() throws HiveException { + if (array.length == rowLength) { + // optimize for common case - just one row for a key, container acts as iterator + return this; + } + return rowLength > 0 ? new RowIterator() : new EmptyRowIterator(-rowLength); + } + + @Override + public List first() throws HiveException { + if (array.length != rowLength) { + throw new AssertionError("Incorrect iterator usage, not single-row"); + } + return this; // optimize for common case - just one row for a key, container acts as row + } + + @Override + public List next() { + return null; // single-row case, there's no next + } + + /** Iterator for row length 0. */ + private static class EmptyRowIterator implements AbstractRowContainer.RowIterator> { + private static final List EMPTY_ROW = new ArrayList(); + private int rowCount; + public EmptyRowIterator(int rowCount) { + this.rowCount = rowCount; + } + + @Override + public List first() throws HiveException { + return next(); + } + + @Override + public List next() throws HiveException { + return (--rowCount < 0) ? null : EMPTY_ROW; + } + } + + /** Row iterator for non-zero-length rows. */ + private class RowIterator implements AbstractRowContainer.RowIterator> { + private int index = 0; + + @Override + public List first() throws HiveException { + index = 0; + if (array.length > 0) { + return new ReadOnlySubList(0, rowLength); + } + return null; + } + + @Override + public List next() { + index += rowLength; + if (index < array.length) { + return new ReadOnlySubList(index, rowLength); + } + return null; + } + } + + private void read(SerDe serde, Writable writable, int rowOffset) throws HiveException { + try { + ObjectInspectorUtils.copyStructToArray( + serde.deserialize(writable), serde.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE, this.array, rowOffset * rowLength); + } catch (SerDeException ex) { + throw new HiveException("Lazy deserialize error", ex); + } + } + + @Override + public int rowCount() throws HiveException { + return rowLength > 0 ? (array.length / rowLength) : -rowLength; // see rowLength javadoc + } + + @Override + public void clearRows() { + array = EMPTY_OBJECT_ARRAY; + rowLength = 0; + } + + @Override + public byte getAliasFilter() throws HiveException { + ensureAliasFilter(); + return this.aliasFilter; + } + + private void ensureAliasFilter() throws HiveException { + if (!isAliasFilterSet && rowLength > 0) { + for (int offset = rowLength - 1; offset < array.length; offset += rowLength) { + aliasFilter &= ((ShortWritable)array[offset]).get(); + } + } + isAliasFilterSet = true; + } + + @Override + public MapJoinRowContainer copy() throws HiveException { + FlatRowContainer result = new FlatRowContainer(); + result.array = new Object[this.array.length]; + System.arraycopy(this.array, 0, result.array, 0, this.array.length); + result.rowLength = rowLength; + result.aliasFilter = aliasFilter; + return result; + } + + // Implementation of List and assorted methods + + private void listRealloc(int length) { + Object[] array = new Object[length]; + if (this.array.length > 0) { + System.arraycopy(this.array, 0, array, 0, this.array.length); + } + this.array = array; + } + + @Override + public int size() { + checkSingleRow(); + return array.length; + } + + @Override + public Object get(int index) { + return array[index]; + } + + private class ReadOnlySubList extends AbstractList { + private int offset; + private int size; + + ReadOnlySubList(int from, int size) { + this.offset = from; + this.size = size; + } + + public Object get(int index) { + return array[index + offset]; + } + + public int size() { + return size; + } + + public Iterator iterator() { + return listIterator(); + } + + public ListIterator listIterator(int index) { + return listIteratorInternal(offset + index, offset, offset + size); + } + + public List subList(int fromIndex, int toIndex) { + return new ReadOnlySubList(offset + fromIndex, toIndex - fromIndex); + } + + public Object[] toArray() { + Object[] result = new Object[size]; + System.arraycopy(array, offset, result, 0, size); + return result; + } + } // end ReadOnlySubList + + @Override + public Object[] toArray() { + checkSingleRow(); + return array; + } + + @Override + public Iterator iterator() { + return listIterator(); + } + + @Override + public ListIterator listIterator() { + return listIterator(0); + } + + @Override + public ListIterator listIterator(final int index) { + checkSingleRow(); + return listIteratorInternal(index, 0, array.length); + } + + private ListIterator listIteratorInternal( + final int index, final int iterMinPos, final int iterMaxPos) { + return new ListIterator() { + private int pos = index - 1; + public int nextIndex() { + return pos + 1; + } + public int previousIndex() { + return pos - 1; + } + public boolean hasNext() { + return nextIndex() < iterMaxPos; + } + public boolean hasPrevious() { + return previousIndex() >= iterMinPos; + } + public Object next() { + if (!hasNext()) throw new NoSuchElementException(); + return get(++pos); + } + public Object previous() { + if (!hasPrevious()) throw new NoSuchElementException(); + return get(--pos); + } + + public void remove() { throw new UnsupportedOperationException(); } + public void set(Object e) { throw new UnsupportedOperationException(); } + public void add(Object e) { throw new UnsupportedOperationException(); } + }; // end ListIterator + } + + @Override + public int indexOf(Object o) { + checkSingleRow(); + for (int i = 0; i < array.length; ++i) { + if (o == null) { + if (array[i] == null) return i; + } else { + if (o.equals(array[i])) return i; + } + } + return -1; + } + + private void checkSingleRow() throws AssertionError { + if (array.length != rowLength) { + throw new AssertionError("Incorrect list usage, not single-row"); + } + } + + @Override + public int lastIndexOf(Object o) { + checkSingleRow(); + for (int i = array.length - 1; i >= 0; --i) { + if (o == null) { + if (array[i] == null) return i; + } else { + if (o.equals(array[i])) return i; + } + } + return -1; + } + + @Override + public List subList(int fromIndex, int toIndex) { + checkSingleRow(); + return new ReadOnlySubList(fromIndex, toIndex - fromIndex); + } + + public boolean addAll(int index, Collection c) { + throw new UnsupportedOperationException(); + } + public Object set(int index, Object element) { throw new UnsupportedOperationException(); } + public void add(int index, Object element) { throw new UnsupportedOperationException(); } + public Object remove(int index) { throw new UnsupportedOperationException(); } +} + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (working copy) @@ -58,8 +58,6 @@ private static final float LOADFACTOR = 0.75f; private final HashMap mHash; // main memory HashMap private MapJoinKey lastKey = null; - private final boolean useLazyRows; - private final boolean useOptimizedKeys; private Output output = new Output(0); // Reusable output for serialization public HashMapWrapper(Map metaData) { @@ -67,30 +65,24 @@ int threshold = Integer.parseInt(metaData.get(THESHOLD_NAME)); float loadFactor = Float.parseFloat(metaData.get(LOAD_NAME)); mHash = new HashMap(threshold, loadFactor); - useLazyRows = useOptimizedKeys = false; } public HashMapWrapper() { this(HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT.defaultFloatVal, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal, - HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false, -1); + HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, -1); } public HashMapWrapper(Configuration hconf, long keyCount) { this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT), HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD), - HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR), - HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINLAZYHASHTABLE), - HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS), keyCount); + HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR), keyCount); } - private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, - boolean useLazyRows, boolean useOptimizedKeys, long keyCount) { + private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount) { super(createConstructorMetaData(threshold, loadFactor)); threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount); mHash = new HashMap(threshold, loadFactor); - this.useLazyRows = useLazyRows; - this.useOptimizedKeys = useOptimizedKeys; } public static int calculateTableSize( @@ -131,21 +123,14 @@ public MapJoinKey putRow(MapJoinObjectSerDeContext keyContext, Writable currentKey, MapJoinObjectSerDeContext valueContext, Writable currentValue) throws SerDeException, HiveException { - // We pass key in as reference, to find out quickly if optimized keys can be used. - // However, we do not reuse the object since we are putting them into the hashmap. - // Later, we don't create optimized keys in MapJoin if hash map doesn't have optimized keys. - if (lastKey == null && !useOptimizedKeys) { - lastKey = new MapJoinKeyObject(); - } - - lastKey = MapJoinKey.read(output, lastKey, keyContext, currentKey, false); - LazyFlatRowContainer values = (LazyFlatRowContainer)get(lastKey); + MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey); + FlatRowContainer values = (FlatRowContainer)get(key); if (values == null) { - values = new LazyFlatRowContainer(); - put(lastKey, values); + values = new FlatRowContainer(); + put(key, values); } - values.add(valueContext, (BytesWritable)currentValue, useLazyRows); - return lastKey; + values.add(valueContext, (BytesWritable)currentValue); + return key; } @Override Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java (working copy) @@ -1,495 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.persistence; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.ObjectOutputStream; -import java.util.AbstractCollection; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; -import java.util.NoSuchElementException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; -import org.apache.hadoop.io.BinaryComparable; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Writable; - -@SuppressWarnings("deprecation") -public class LazyFlatRowContainer extends AbstractCollection - implements MapJoinRowContainer, AbstractRowContainer.RowIterator>, List { - private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; - private static final int UNKNOWN = Integer.MAX_VALUE; - - private static Log LOG = LogFactory.getLog(LazyFlatRowContainer.class); - - /** - * In lazy mode, 0s element contains context for deserialization and all the other - * elements contains byte arrays to be deserialized. After deserialization, the array - * contains row count * row size elements - a matrix of rows stored. - */ - private Object[] array; - /** - * This is kind of tricky. UNKNOWN number means lazy mode. Other positive numbers represent - * row length (see array javadoc). Non-positive numbers mean row length is zero (thus, - * array is empty); they represent (negated) number of rows (for joins w/o projections). - */ - private int rowLength = UNKNOWN; - private byte aliasFilter = (byte) 0xff; - - public LazyFlatRowContainer() { - this.array = EMPTY_OBJECT_ARRAY; - } - - /** Called when loading the hashtable. */ - public void add(MapJoinObjectSerDeContext context, - BytesWritable value, boolean allowLazy) throws HiveException { - if (allowLazy) { - addLazy(context, value); - return; - } - SerDe serde = context.getSerDe(); - boolean hasValues = isLazy() ? setRowLength(serde, 0) : (rowLength > 0); - int rowCount = rowCount(); - if (hasValues) { - listRealloc(array.length + rowLength); - read(serde, value, rowCount); - } else { - --rowLength; // see rowLength javadoc - } - } - - private void addLazy(MapJoinObjectSerDeContext valueContext, BytesWritable currentValue) { - if (!isLazy()) { - throw new AssertionError("Not in lazy mode"); - } - int size = this.array.length; - if (size == 0) { - // TODO: we store valueContext needlessly in each RowContainer because the - // accessor cannot pass it to us for lazy deserialization. - listRealloc(2); - this.array[0] = valueContext; - ++size; - } else { - if (this.array[0] != valueContext) { - throw new AssertionError("Different valueContext for the same table"); - } - listRealloc(size + 1); - } - byte[] rawData = new byte[currentValue.getSize()]; - System.arraycopy(currentValue.getBytes(), 0, rawData, 0, rawData.length); - this.array[size] = rawData; - } - - // Implementation of AbstractRowContainer and assorted methods - - @Override - public void addRow(List t) throws HiveException { - LOG.debug("Add is called with " + t.size() + " objects"); - // This is not called when building HashTable; we don't expect it to be called ever. - int offset = prepareForAdd(t.size()); - if (offset < 0) return; - for (int i = 0; i < t.size(); ++i) { - this.array[offset + i] = t.get(i); - } - } - - @Override - public void addRow(Object[] value) throws HiveException { - LOG.debug("Add is called with " + value.length + " objects"); - // This is not called when building HashTable; we don't expect it to be called ever. - int offset = prepareForAdd(value.length); - if (offset < 0) return; - System.arraycopy(value, 0, this.array, offset, value.length); - } - - private int prepareForAdd(int len) throws HiveException { - if (isLazy()) { - throw new AssertionError("Cannot add in lazy mode"); - } - if (rowLength < 0) { - if (len != 0) { - throw new HiveException("Different size rows: 0 and " + len); - } - --rowLength; // see rowLength javadoc - return -1; - } - if (rowLength != len) { - throw new HiveException("Different size rows: " + rowLength + " and " + len); - } - int oldLen = this.array.length; - listRealloc(oldLen + len); - return oldLen; - } - - @Override - public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out) { - throw new UnsupportedOperationException(this.getClass().getName() + " cannot be serialized"); - } - - @Override - public AbstractRowContainer.RowIterator> rowIter() throws HiveException { - ensureEager(); // if someone wants an iterator they are probably going to use it - if (array.length == rowLength) { - // optimize for common case - just one row for a key, container acts as iterator - return this; - } - return rowLength > 0 ? new RowIterator() : new EmptyRowIterator(-rowLength); - } - - @Override - public List first() throws HiveException { - if (isLazy()) { - throw new AssertionError("In lazy mode"); - } - if (array.length != rowLength) { - throw new AssertionError("Incorrect iterator usage, not single-row"); - } - return this; // optimize for common case - just one row for a key, container acts as row - } - - @Override - public List next() { - return null; // single-row case, there's no next - } - - /** Iterator for row length 0. */ - private static class EmptyRowIterator implements AbstractRowContainer.RowIterator> { - private static final List EMPTY_ROW = new ArrayList(); - private int rowCount; - public EmptyRowIterator(int rowCount) { - this.rowCount = rowCount; - } - - @Override - public List first() throws HiveException { - return next(); - } - - @Override - public List next() throws HiveException { - return (--rowCount < 0) ? null : EMPTY_ROW; - } - } - - /** Row iterator for non-zero-length rows. */ - private class RowIterator implements AbstractRowContainer.RowIterator> { - private int index = 0; - - @Override - public List first() throws HiveException { - assert !isLazy(); - index = 0; - if (array.length > 0) { - return new ReadOnlySubList(0, rowLength); - } - return null; - } - - @Override - public List next() { - assert !isLazy(); - index += rowLength; - if (index < array.length) { - return new ReadOnlySubList(index, rowLength); - } - return null; - } - } - - private void ensureEager() throws HiveException { - if (!isLazy()) return; - if (this.array.length == 0) { - rowLength = 0; - return; - } - Object[] lazyObjs = this.array; - assert lazyObjs.length > 1; - MapJoinObjectSerDeContext context = (MapJoinObjectSerDeContext)lazyObjs[0]; - SerDe serde = context.getSerDe(); - int rowCount = lazyObjs.length - 1; - if (!setRowLength(serde, rowCount)) return; - - this.array = new Object[rowLength * rowCount]; - ByteBufferWritable writable = new ByteBufferWritable(); - for (int i = 0; i < rowCount; ++i) { - writable.setBytes((byte[])lazyObjs[i + 1]); - read(serde, writable, i); - } - setAliasFilter(context); - } - - private boolean setRowLength(SerDe serde, int rowCount) throws HiveException { - try { - rowLength = ObjectInspectorUtils.getStructSize(serde.getObjectInspector()); - } catch (SerDeException ex) { - throw new HiveException("Get structure size error", ex); - } - if (rowLength == 0) { - rowLength = -rowCount; // see javadoc for rowLength - array = EMPTY_OBJECT_ARRAY; - return false; - } - return true; - } - - private void read(SerDe serde, Writable writable, int rowOffset) throws HiveException { - try { - ObjectInspectorUtils.copyStructToArray( - serde.deserialize(writable), serde.getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE, this.array, rowOffset * rowLength); - } catch (SerDeException ex) { - throw new HiveException("Lazy deserialize error", ex); - } - } - - private boolean isLazy() { - return rowLength == UNKNOWN; - } - - @Override - public int rowCount() throws HiveException { - ensureEager(); - return rowLength > 0 ? (array.length / rowLength) : -rowLength; // see rowLength javadoc - } - - @Override - public void clearRows() { - assert !isLazy(); - array = EMPTY_OBJECT_ARRAY; - rowLength = 0; - } - - @Override - public byte getAliasFilter() throws HiveException { - ensureEager(); - return this.aliasFilter; - } - - private void setAliasFilter(MapJoinObjectSerDeContext context) throws HiveException { - if (isLazy()) { - throw new AssertionError("In lazy mode"); - } - if (rowLength <= 0 || !context.hasFilterTag()) return; - for (int offset = rowLength - 1; offset < array.length; offset += rowLength) { - aliasFilter &= ((ShortWritable)array[offset]).get(); - } - } - - @Override - public MapJoinRowContainer copy() throws HiveException { - ensureEager(); // If someone wants a copy they are probably going to use it. - LazyFlatRowContainer result = new LazyFlatRowContainer(); - result.array = new Object[this.array.length]; - System.arraycopy(this.array, 0, result.array, 0, this.array.length); - result.rowLength = rowLength; - result.aliasFilter = aliasFilter; - return result; - } - - // Implementation of List and assorted methods - - private void listRealloc(int length) { - Object[] array = new Object[length]; - if (this.array.length > 0) { - System.arraycopy(this.array, 0, array, 0, this.array.length); - } - this.array = array; - } - - @Override - public int size() { - checkSingleRow(); - return array.length; - } - - @Override - public Object get(int index) { - return array[index]; - } - - private class ReadOnlySubList extends AbstractList { - private int offset; - private int size; - - ReadOnlySubList(int from, int size) { - this.offset = from; - this.size = size; - } - - public Object get(int index) { - return array[index + offset]; - } - - public int size() { - return size; - } - - public Iterator iterator() { - return listIterator(); - } - - public ListIterator listIterator(int index) { - return listIteratorInternal(offset + index, offset, offset + size); - } - - public List subList(int fromIndex, int toIndex) { - return new ReadOnlySubList(offset + fromIndex, toIndex - fromIndex); - } - - public Object[] toArray() { - Object[] result = new Object[size]; - System.arraycopy(array, offset, result, 0, size); - return result; - } - } // end ReadOnlySubList - - @Override - public Object[] toArray() { - checkSingleRow(); - return array; - } - - @Override - public Iterator iterator() { - return listIterator(); - } - - @Override - public ListIterator listIterator() { - return listIterator(0); - } - - @Override - public ListIterator listIterator(final int index) { - checkSingleRow(); - return listIteratorInternal(index, 0, array.length); - } - - private ListIterator listIteratorInternal( - final int index, final int iterMinPos, final int iterMaxPos) { - return new ListIterator() { - private int pos = index - 1; - public int nextIndex() { - return pos + 1; - } - public int previousIndex() { - return pos - 1; - } - public boolean hasNext() { - return nextIndex() < iterMaxPos; - } - public boolean hasPrevious() { - return previousIndex() >= iterMinPos; - } - public Object next() { - if (!hasNext()) throw new NoSuchElementException(); - return get(++pos); - } - public Object previous() { - if (!hasPrevious()) throw new NoSuchElementException(); - return get(--pos); - } - - public void remove() { throw new UnsupportedOperationException(); } - public void set(Object e) { throw new UnsupportedOperationException(); } - public void add(Object e) { throw new UnsupportedOperationException(); } - }; // end ListIterator - } - - /** Fake writable that can be reset with different bytes. */ - private static class ByteBufferWritable extends BinaryComparable implements Writable { - byte[] bytes = null; - - @Override - public byte[] getBytes() { - return bytes; - } - - @Override - public int getLength() { - return bytes.length; - } - - public void setBytes(byte[] bytes) { - this.bytes = bytes; - } - - public void readFields(DataInput arg0) { throw new UnsupportedOperationException(); } - public void write(DataOutput arg0) { throw new UnsupportedOperationException(); } - } // end ByteBufferWritable - - - @Override - public int indexOf(Object o) { - checkSingleRow(); - for (int i = 0; i < array.length; ++i) { - if (o == null) { - if (array[i] == null) return i; - } else { - if (o.equals(array[i])) return i; - } - } - return -1; - } - - private void checkSingleRow() throws AssertionError { - if (array.length != rowLength) { - throw new AssertionError("Incorrect list usage, not single-row"); - } - } - - @Override - public int lastIndexOf(Object o) { - checkSingleRow(); - for (int i = array.length - 1; i >= 0; --i) { - if (o == null) { - if (array[i] == null) return i; - } else { - if (o.equals(array[i])) return i; - } - } - return -1; - } - - @Override - public List subList(int fromIndex, int toIndex) { - checkSingleRow(); - return new ReadOnlySubList(fromIndex, toIndex - fromIndex); - } - - public boolean addAll(int index, Collection c) { - throw new UnsupportedOperationException(); - } - public Object set(int index, Object element) { throw new UnsupportedOperationException(); } - public void add(int index, Object element) { throw new UnsupportedOperationException(); } - public Object remove(int index) { throw new UnsupportedOperationException(); } -} - Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (working copy) @@ -57,22 +57,11 @@ public abstract boolean hasAnyNulls(int fieldCount, boolean[] nullsafes); @SuppressWarnings("deprecation") - public static MapJoinKey read(Output output, MapJoinKey key, - MapJoinObjectSerDeContext context, Writable writable, boolean mayReuseKey) - throws SerDeException, HiveException { + public static MapJoinKey read(Output output, MapJoinObjectSerDeContext context, + Writable writable) throws SerDeException, HiveException { SerDe serde = context.getSerDe(); Object obj = serde.deserialize(writable); - boolean useOptimized = useOptimizedKeyBasedOnPrev(key); - if (useOptimized || key == null) { - byte[] structBytes = serialize(output, obj, serde.getObjectInspector(), !useOptimized); - if (structBytes != null) { - return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes); - } else if (useOptimized) { - throw new SerDeException( - "Failed to serialize " + obj + " even though optimized keys are used"); - } - } - MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject(); + MapJoinKeyObject result = new MapJoinKeyObject(); result.read(serde.getObjectInspector(), obj); return result; } @@ -98,35 +87,6 @@ SUPPORTED_PRIMITIVES.add(PrimitiveCategory.CHAR); } - private static byte[] serialize(Output byteStream, - Object obj, ObjectInspector oi, boolean checkTypes) throws HiveException { - if (null == obj || !(oi instanceof StructObjectInspector)) { - return null; // not supported - } - StructObjectInspector soi = (StructObjectInspector)oi; - List fields = soi.getAllStructFieldRefs(); - int size = fields.size(); - if (size > 8) { - return null; // not supported - } else if (size == 0) { - return EMPTY_BYTE_ARRAY; // shortcut for null keys - } - Object[] fieldData = new Object[size]; - List fieldOis = new ArrayList(size); - for (int i = 0; i < size; ++i) { - StructField field = fields.get(i); - ObjectInspector foi = field.getFieldObjectInspector(); - if (checkTypes && !isSupportedField(foi)) { - return null; - } - fieldData[i] = soi.getStructFieldData(obj, field); - fieldOis.add(foi); - } - - byteStream = serializeRow(byteStream, fieldData, fieldOis, null); - return Arrays.copyOf(byteStream.getData(), byteStream.getLength()); - } - public static boolean isSupportedField(ObjectInspector foi) { if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory(); @@ -136,19 +96,6 @@ public static MapJoinKey readFromVector(Output output, MapJoinKey key, Object[] keyObject, List keyOIs, boolean mayReuseKey) throws HiveException { - boolean useOptimized = useOptimizedKeyBasedOnPrev(key); - if (useOptimized || key == null) { - if (keyObject.length <= 8) { - output = serializeRow(output, keyObject, keyOIs, null); - return MapJoinKeyBytes.fromBytes(key, mayReuseKey, - Arrays.copyOf(output.getData(), output.getLength())); - } - if (useOptimized) { - throw new HiveException( - "Failed to serialize " + Arrays.toString(keyObject) + - " even though optimized keys are used"); - } - } MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject(); result.setKeyObjects(keyObject); return result; @@ -178,32 +125,11 @@ public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object[] keyObject, List keyFieldsOI, boolean mayReuseKey) throws HiveException { - boolean useOptimized = useOptimizedKeyBasedOnPrev(key); - if (useOptimized || key == null) { - if (keyObject.length <= 8) { - byte[] structBytes; - if (keyObject.length == 0) { - structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys - } else { - output = serializeRow(output, keyObject, keyFieldsOI, null); - structBytes = Arrays.copyOf(output.getData(), output.getLength()); - } - return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes); - } - if (useOptimized) { - throw new HiveException( - "Failed to serialize " + Arrays.toString(keyObject) + - " even though optimized keys are used"); - } - } MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject(); result.readFromRow(keyObject, keyFieldsOI); return result; } - private static final Log LOG = LogFactory.getLog(MapJoinKey.class); - - /** * Serializes row to output. * @param byteStream Output to reuse. Can be null, in that case a new one would be created. @@ -228,8 +154,4 @@ } return byteStream; } - - private static boolean useOptimizedKeyBasedOnPrev(MapJoinKey key) { - return (key != null) && (key instanceof MapJoinKeyBytes); - } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyBytes.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyBytes.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyBytes.java (working copy) @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.persistence; - -import java.io.ObjectOutputStream; -import java.util.Arrays; - -import org.apache.hadoop.util.hash.MurmurHash; - -/** - * Size-optimized implementation of MapJoinKeyBase. MJK only needs to support equality and - * hashCode, so for simple cases we can write the requisite writables that are part of the - * key into byte array and retain the functionality without storing the writables themselves. - */ -@SuppressWarnings("deprecation") -public class MapJoinKeyBytes extends MapJoinKey { - private static final MurmurHash hash = (MurmurHash)MurmurHash.getInstance(); - /** - * First byte is field count. The rest is written using BinarySortableSerDe. - */ - private byte[] array; - - private void setBytes(byte[] array) { - this.array = array; - } - - @Override - public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) { - throw new UnsupportedOperationException(this.getClass().getName() + " cannot be serialized"); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || !(obj instanceof MapJoinKeyBytes)) return false; - MapJoinKeyBytes other = (MapJoinKeyBytes)obj; - return Arrays.equals(this.array, other.array); - } - - @Override - public int hashCode() { - return hash.hash(array); - } - - @Override - public boolean hasAnyNulls(int fieldCount, boolean[] nullsafes) { - if (this.array.length == 0) return false; // null key - byte nulls = (byte)(this.array[0]); - for (int i = 0; i < fieldCount; ++i) { - if (((nulls & 1) == 0) && (nullsafes == null || !nullsafes[i])) return true; - nulls >>>= 1; - } - return false; - } - - public static MapJoinKey fromBytes(MapJoinKey key, boolean mayReuseKey, byte[] structBytes) { - MapJoinKeyBytes result = (mayReuseKey && key != null) - ? (MapJoinKeyBytes)key : new MapJoinKeyBytes(); - result.setBytes(structBytes); - return result; - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (working copy) @@ -57,8 +57,6 @@ private ExecMapperContext context; private Configuration hconf; private MapJoinDesc desc; - private MapJoinKey lastKey = null; - private int rowCount = 0; @Override public void init(ExecMapperContext context, Configuration hconf, MapJoinOperator joinOp) { @@ -111,8 +109,7 @@ : new HashMapWrapper(hconf, keyCount); while (kvReader.next()) { - rowCount++; - lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(), + tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(), valCtx, (Writable)kvReader.getCurrentValue()); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java (working copy) @@ -70,6 +70,7 @@ * Key vector expressions. */ private VectorExpression[] keyExpressions; + private int outputKeyLength; private boolean isVectorOutput; @@ -768,9 +769,16 @@ List keysDesc = conf.getKeys(); try { - keyOutputWriters = new VectorExpressionWriter[keyExpressions.length]; + List outputFieldNames = conf.getOutputColumnNames(); - for(int i = 0; i < keyExpressions.length; ++i) { + // grouping id should be pruned, which is the last of key columns + // see ColumnPrunerGroupByProc + outputKeyLength = + conf.pruneGroupingSetId() ? keyExpressions.length - 1 : keyExpressions.length; + + keyOutputWriters = new VectorExpressionWriter[outputKeyLength]; + + for(int i = 0; i < outputKeyLength; ++i) { keyOutputWriters[i] = VectorExpressionWriterFactory. genVectorExpressionWritable(keysDesc.get(i)); objectInspectors.add(keyOutputWriters[i].getObjectInspector()); @@ -788,7 +796,6 @@ aggregationBatchInfo.compileAggregationBatchInfo(aggregators); } LOG.warn("VectorGroupByOperator is vector output " + isVectorOutput); - List outputFieldNames = conf.getOutputColumnNames(); outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector( outputFieldNames, objectInspectors); if (isVectorOutput) { @@ -807,9 +814,9 @@ initializeChildren(hconf); - forwardCache = new Object[keyExpressions.length + aggregators.length]; + forwardCache = new Object[outputKeyLength + aggregators.length]; - if (keyExpressions.length == 0) { + if (outputKeyLength == 0) { processingMode = this.new ProcessingModeGlobalAggregate(); } else if (conf.getVectorDesc().isVectorGroupBatches()) { // Sorted GroupBy of vector batches where an individual batch has the same group key (e.g. reduce). @@ -872,7 +879,7 @@ int fi = 0; if (!isVectorOutput) { // Output row. - for (int i = 0; i < keyExpressions.length; ++i) { + for (int i = 0; i < outputKeyLength; ++i) { forwardCache[fi++] = keyWrappersBatch.getWritableKeyValue ( kw, i, keyOutputWriters[i]); } @@ -886,7 +893,7 @@ forward(forwardCache, outputObjInspector); } else { // Output keys and aggregates into the output batch. - for (int i = 0; i < keyExpressions.length; ++i) { + for (int i = 0; i < outputKeyLength; ++i) { vectorColumnAssign[fi++].assignObjectValue(keyWrappersBatch.getWritableKeyValue ( kw, i, keyOutputWriters[i]), outputBatch.size); } @@ -910,7 +917,7 @@ */ private void writeGroupRow(VectorAggregationBufferRow agg, DataOutputBuffer buffer) throws HiveException { - int fi = keyExpressions.length; // Start after group keys. + int fi = outputKeyLength; // Start after group keys. for (int i = 0; i < aggregators.length; ++i) { vectorColumnAssign[fi++].assignObjectValue(aggregators[i].evaluateOutput( agg.getAggregationBuffer(i)), outputBatch.size); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringInitCap.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringInitCap.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringInitCap.java (working copy) @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.commons.lang.WordUtils; +import org.apache.hadoop.io.Text; + +/** + * Returns str, with the first letter of each word in uppercase, all other + * letters in lowercase. Words are delimited by white space. e.g. initcap('the + * soap') returns 'The Soap' + * Extends {@link StringUnaryUDF}. + */ +public class StringInitCap extends StringUnaryUDF { + private static final long serialVersionUID = 1L; + + public StringInitCap(int colNum, int outputColumn) { + super(colNum, outputColumn, new IUDFUnaryString() { + + Text t = new Text(); + + @Override + public Text evaluate(Text s) { + if (s == null) { + return null; + } + t.set(WordUtils.capitalizeFully(s.toString())); + return t; + } + }); + } + + public StringInitCap() { + super(); + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java (working copy) @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Redactor implements Hook, Configurable { + + private Configuration conf; + + /** + * Guranteed to be called before redactQuery is called + */ + public void setConf(Configuration conf) { + this.conf = conf; + } + + public Configuration getConf() { + return conf; + } + + /** + * Implementations may modify the query so that when placed in the job.xml + * and thus potenially exposed to admin users, the query does not expose + * sensitive information. + */ + public String redactQuery(String query) { + return query; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java (working copy) @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable; @@ -98,7 +99,7 @@ FileSystem fs = indexFilePath.getFileSystem(conf); FileStatus indexStat = fs.getFileStatus(indexFilePath); if (indexStat.isDir()) { - FileStatus[] fss = fs.listStatus(indexFilePath); + FileStatus[] fss = fs.listStatus(indexFilePath, FileUtils.HIDDEN_FILES_PATH_FILTER); for (FileStatus f : fss) { paths.add(f.getPath()); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java (working copy) @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -92,14 +91,7 @@ List errors = new ArrayList(); FileSystem fs = dir.getFileSystem(job); - FileStatus[] matches = fs.globStatus(dir, new PathFilter() { - - @Override - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - }); + FileStatus[] matches = fs.globStatus(dir, FileUtils.HIDDEN_FILES_PATH_FILTER); if (matches == null) { errors.add(new IOException("Input path does not exist: " + dir)); } else if (matches.length == 0) { @@ -113,7 +105,8 @@ if (!errors.isEmpty()) { throw new InvalidInputException(errors); } - LOG.info("Total input paths to process : " + result.size()); + LOG.debug("Matches for " + dir + ": " + result); + LOG.info("Total input paths to process : " + result.size() + " from dir " + dir); return result.toArray(new FileStatus[result.size()]); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (working copy) @@ -30,6 +30,10 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -38,6 +42,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -74,6 +79,48 @@ private static final String CLASS_NAME = CombineHiveInputFormat.class.getName(); public static final Log LOG = LogFactory.getLog(CLASS_NAME); + // max number of threads we can use to check non-combinable paths + private static final int MAX_CHECK_NONCOMBINABLE_THREAD_NUM = 50; + private static final int DEFAULT_NUM_PATH_PER_THREAD = 100; + + private class CheckNonCombinablePathCallable implements Callable> { + private final Path[] paths; + private final int start; + private final int length; + private final JobConf conf; + + public CheckNonCombinablePathCallable(Path[] paths, int start, int length, JobConf conf) { + this.paths = paths; + this.start = start; + this.length = length; + this.conf = conf; + } + + @Override + public Set call() throws Exception { + Set nonCombinablePathIndices = new HashSet(); + for (int i = 0; i < length; i++) { + PartitionDesc part = + HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, paths[i + start], + IOPrepareCache.get().allocatePartitionDescMap()); + // Use HiveInputFormat if any of the paths is not splittable + Class inputFormatClass = part.getInputFileFormatClass(); + InputFormat inputFormat = + getInputFormatFromCache(inputFormatClass, conf); + if (inputFormat instanceof AvoidSplitCombination && + ((AvoidSplitCombination) inputFormat).shouldSkipCombine(paths[i + start], conf)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The path [" + paths[i + start] + + "] is being parked for HiveInputFormat.getSplits"); + } + nonCombinablePathIndices.add(i); + } + } + return nonCombinablePathIndices; + } + } + /** * CombineHiveInputSplit encapsulates an InputSplit with its corresponding * inputFormatClassName. A CombineHiveInputSplit comprises of multiple chunks @@ -278,8 +325,6 @@ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map pathToPartitionInfo) throws IOException { - PerfLogger perfLogger = PerfLogger.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); init(job); Map> pathToAliases = mrwork.getPathToAliases(); Map> aliasToWork = @@ -290,7 +335,6 @@ InputSplit[] splits = null; if (combine == null) { splits = super.getSplits(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return splits; } @@ -349,13 +393,12 @@ } else if ((new CompressionCodecFactory(job)).getCodec(path) != null) { //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) splits = super.getSplits(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return splits; } while (dirs.peek() != null) { Path tstPath = dirs.remove(); - FileStatus[] fStatus = inpFs.listStatus(tstPath); + FileStatus[] fStatus = inpFs.listStatus(tstPath, FileUtils.HIDDEN_FILES_PATH_FILTER); for (int idx = 0; idx < fStatus.length; idx++) { if (fStatus[idx].isDir()) { dirs.offer(fStatus[idx].getPath()); @@ -363,7 +406,6 @@ fStatus[idx].getPath()) != null) { //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) splits = super.getSplits(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return splits; } } @@ -373,7 +415,6 @@ //don't combine if inputformat is a SymlinkTextInputFormat if (inputFormat instanceof SymlinkTextInputFormat) { splits = super.getSplits(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return splits; } @@ -451,7 +492,6 @@ } LOG.info("number of splits " + result.size()); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return result.toArray(new CombineHiveInputSplit[result.size()]); } @@ -460,6 +500,8 @@ */ @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); init(job); ArrayList result = new ArrayList(); @@ -469,26 +511,37 @@ List nonCombinablePaths = new ArrayList(paths.length / 2); List combinablePaths = new ArrayList(paths.length / 2); - for (Path path : paths) { - - PartitionDesc part = - HiveFileFormatUtils.getPartitionDescFromPathRecursively( - pathToPartitionInfo, path, - IOPrepareCache.get().allocatePartitionDescMap()); - - // Use HiveInputFormat if any of the paths is not splittable - Class inputFormatClass = part.getInputFileFormatClass(); - InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); - if (inputFormat instanceof AvoidSplitCombination && - ((AvoidSplitCombination) inputFormat).shouldSkipCombine(path, job)) { - if (LOG.isDebugEnabled()) { - LOG.debug("The split [" + path + - "] is being parked for HiveInputFormat.getSplits"); + int numThreads = Math.min(MAX_CHECK_NONCOMBINABLE_THREAD_NUM, + (int) Math.ceil((double) paths.length / DEFAULT_NUM_PATH_PER_THREAD)); + int numPathPerThread = (int) Math.ceil((double) paths.length / numThreads); + LOG.info("Total number of paths: " + paths.length + + ", launching " + numThreads + " threads to check non-combinable ones."); + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + List>> futureList = new ArrayList>>(numThreads); + try { + for (int i = 0; i < numThreads; i++) { + int start = i * numPathPerThread; + int length = i != numThreads - 1 ? numPathPerThread : paths.length - start; + futureList.add(executor.submit( + new CheckNonCombinablePathCallable(paths, start, length, job))); + } + Set nonCombinablePathIndices = new HashSet(); + for (Future> future : futureList) { + nonCombinablePathIndices.addAll(future.get()); + } + for (int i = 0; i < paths.length; i++) { + if (nonCombinablePathIndices.contains(i)) { + nonCombinablePaths.add(paths[i]); + } else { + combinablePaths.add(paths[i]); } - nonCombinablePaths.add(path); - } else { - combinablePaths.add(path); } + } catch (Exception e) { + LOG.error("Error checking non-combinable path", e); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); + throw new IOException(e); + } finally { + executor.shutdownNow(); } // Store the previous value for the path specification @@ -528,6 +581,7 @@ job.set(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname, oldPaths); } LOG.info("Number of all splits " + result.size()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return result.toArray(new InputSplit[result.size()]); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -58,7 +59,7 @@ if (!fStatus.isDir()) { symlinks = new FileStatus[] { fStatus }; } else { - symlinks = fileSystem.listStatus(symlinkDir); + symlinks = fileSystem.listStatus(symlinkDir, FileUtils.HIDDEN_FILES_PATH_FILTER); } toRemovePaths.add(path); ArrayList aliases = pathToAliases.remove(path); Index: ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java (working copy) @@ -23,19 +23,15 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; -import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; @@ -192,7 +188,7 @@ List targetPaths, List symlinkPaths) throws IOException { for (Path symlinkDir : symlinksDirs) { FileSystem fileSystem = symlinkDir.getFileSystem(conf); - FileStatus[] symlinks = fileSystem.listStatus(symlinkDir); + FileStatus[] symlinks = fileSystem.listStatus(symlinkDir, FileUtils.HIDDEN_FILES_PATH_FILTER); // Read paths from each symlink file. for (FileStatus symlink : symlinks) { Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java (working copy) @@ -254,9 +254,13 @@ private HiveLockMode getWriteEntityLockMode (WriteEntity we) { HiveLockMode lockMode = we.isComplete() ? HiveLockMode.EXCLUSIVE : HiveLockMode.SHARED; - //but the writeEntity is complete in DDL operations, and we need check its writeType to - //to determine the lockMode - switch (we.getWriteType()) { + //but the writeEntity is complete in DDL operations, instead DDL sets the writeType, so + //we use it to determine its lockMode, and first we check if the writeType was set + WriteEntity.WriteType writeType = we.getWriteType(); + if (writeType == null) { + return lockMode; + } + switch (writeType) { case DDL_EXCLUSIVE: return HiveLockMode.EXCLUSIVE; case DDL_SHARED: Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java (working copy) @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.lockmgr.zookeeper; + +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; + +public class CuratorFrameworkSingleton { + private static HiveConf conf = null; + private static CuratorFramework sharedClient = null; + static final Log LOG = LogFactory.getLog("CuratorFrameworkSingleton"); + static { + // Add shutdown hook. + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + closeAndReleaseInstance(); + } + }); + } + + public static synchronized CuratorFramework getInstance(HiveConf hiveConf) { + if (sharedClient == null) { + // Create a client instance + if (hiveConf == null) { + conf = new HiveConf(); + } else { + conf = hiveConf; + } + int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS); + int baseSleepTime = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS); + int maxRetries = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES); + String quorumServers = ZooKeeperHiveHelper.getQuorumServers(conf); + + sharedClient = CuratorFrameworkFactory.builder().connectString(quorumServers) + .sessionTimeoutMs(sessionTimeout) + .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)) + .build(); + sharedClient.start(); + } + + return sharedClient; + } + + public static synchronized void closeAndReleaseInstance() { + if (sharedClient != null) { + sharedClient.close(); + sharedClient = null; + String shutdownMsg = "Closing ZooKeeper client."; + LOG.info(shutdownMsg); + } + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (working copy) @@ -27,15 +27,10 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; +import org.apache.curator.framework.CuratorFramework; -import java.io.IOException; import java.net.InetAddress; import java.util.*; import java.util.concurrent.TimeUnit; @@ -47,14 +42,11 @@ public static final Log LOG = LogFactory.getLog("ZooKeeperHiveLockManager"); static final private LogHelper console = new LogHelper(LOG); - private ZooKeeper zooKeeper; + private static CuratorFramework curatorFramework; // All the locks are created under this parent private String parent; - private int sessionTimeout; - private String quorumServers; - private long sleepTime; private int numRetriesForLock; private int numRetriesForUnLock; @@ -80,8 +72,6 @@ public void setContext(HiveLockManagerCtx ctx) throws LockException { this.ctx = ctx; HiveConf conf = ctx.getConf(); - sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); - quorumServers = ZooKeeperHiveHelper.getQuorumServers(conf); sleepTime = conf.getTimeVar( HiveConf.ConfVars.HIVE_LOCK_SLEEP_BETWEEN_RETRIES, TimeUnit.MILLISECONDS); @@ -89,20 +79,18 @@ numRetriesForUnLock = conf.getIntVar(HiveConf.ConfVars.HIVE_UNLOCK_NUMRETRIES); try { - renewZookeeperInstance(sessionTimeout, quorumServers); + curatorFramework = CuratorFrameworkSingleton.getInstance(conf); parent = conf.getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_NAMESPACE); - - try { - zooKeeper.create("/" + parent, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException e) { + try{ + curatorFramework.create().withMode(CreateMode.PERSISTENT).forPath("/" + parent, new byte[0]); + } catch (Exception e) { // ignore if the parent already exists - if (e.code() != KeeperException.Code.NODEEXISTS) { + if (!(e instanceof KeeperException) || ((KeeperException)e).code() != KeeperException.Code.NODEEXISTS) { LOG.warn("Unexpected ZK exception when creating parent node /" + parent, e); } } - } catch (Exception e) { - LOG.error("Failed to create ZooKeeper object: ", e); + LOG.error("Failed to create curatorFramework object: ", e); throw new LockException(ErrorMsg.ZOOKEEPER_CLIENT_COULD_NOT_BE_INITIALIZED.getMsg()); } } @@ -116,15 +104,6 @@ numRetriesForUnLock = conf.getIntVar(HiveConf.ConfVars.HIVE_UNLOCK_NUMRETRIES); } - private void renewZookeeperInstance(int sessionTimeout, String quorumServers) - throws InterruptedException, IOException { - if (zooKeeper != null) { - return; - } - - zooKeeper = new ZooKeeper(quorumServers, sessionTimeout, new ZooKeeperHiveHelper.DummyWatcher()); - } - /** * @param key object to be locked * Get the name of the last string. For eg. if you need to lock db/T/ds=1=/hr=1, @@ -266,8 +245,8 @@ * @throws InterruptedException **/ private String createChild(String name, byte[] data, CreateMode mode) - throws KeeperException, InterruptedException { - return zooKeeper.create(name, data, Ids.OPEN_ACL_UNSAFE, mode); + throws Exception { + return curatorFramework.create().withMode(mode).forPath(name, data); } private String getLockName(String parent, HiveLockMode mode) { @@ -347,7 +326,7 @@ private ZooKeeperHiveLock lockPrimitive(HiveLockObject key, HiveLockMode mode, boolean keepAlive, boolean parentCreated, Set conflictingLocks) - throws KeeperException, InterruptedException { + throws Exception { String res; // If the parents have already been created, create the last child only @@ -369,8 +348,8 @@ for (String name : names) { try { res = createChild(name, new byte[0], CreateMode.PERSISTENT); - } catch (KeeperException e) { - if (e.code() != KeeperException.Code.NODEEXISTS) { + } catch (Exception e) { + if (!(e instanceof KeeperException) || ((KeeperException)e).code() != KeeperException.Code.NODEEXISTS) { //if the exception is not 'NODEEXISTS', re-throw it throw e; } @@ -383,11 +362,11 @@ int seqNo = getSequenceNumber(res, getLockName(lastName, mode)); if (seqNo == -1) { - zooKeeper.delete(res, -1); + curatorFramework.delete().forPath(res); return null; } - List children = zooKeeper.getChildren(lastName, false); + List children = curatorFramework.getChildren().forPath(lastName); String exLock = getLockName(lastName, HiveLockMode.EXCLUSIVE); String shLock = getLockName(lastName, HiveLockMode.SHARED); @@ -407,12 +386,11 @@ if ((childSeq >= 0) && (childSeq < seqNo)) { try { - zooKeeper.delete(res, -1); + curatorFramework.delete().forPath(res); } finally { if (LOG.isDebugEnabled()) { - Stat stat = new Stat(); try { - String data = new String(zooKeeper.getData(child, false, stat)); + String data = new String(curatorFramework.getData().forPath(child)); conflictingLocks.add(data); } catch (Exception e) { //ignored @@ -428,11 +406,10 @@ /* Remove the lock specified */ public void unlock(HiveLock hiveLock) throws LockException { - unlockWithRetry(ctx.getConf(), zooKeeper, hiveLock, parent); + unlockWithRetry(hiveLock, parent); } - private void unlockWithRetry(HiveConf conf, ZooKeeper zkpClient, - HiveLock hiveLock, String parent) throws LockException { + private void unlockWithRetry(HiveLock hiveLock, String parent) throws LockException { int tryNum = 0; do { @@ -440,14 +417,13 @@ tryNum++; if (tryNum > 1) { Thread.sleep(sleepTime); - prepareRetry(); } - unlockPrimitive(conf, zkpClient, hiveLock, parent); + unlockPrimitive(hiveLock, parent, curatorFramework); break; } catch (Exception e) { if (tryNum >= numRetriesForUnLock) { String name = ((ZooKeeperHiveLock)hiveLock).getPath(); - LOG.error("Node " + name + " can not be deleted after " + numRetriesForUnLock + " attempts."); + LOG.error("Node " + name + " can not be deleted after " + numRetriesForUnLock + " attempts."); throw new LockException(e); } } @@ -458,21 +434,20 @@ /* Remove the lock specified */ @VisibleForTesting - static void unlockPrimitive(HiveConf conf, ZooKeeper zkpClient, - HiveLock hiveLock, String parent) throws LockException { + static void unlockPrimitive(HiveLock hiveLock, String parent, CuratorFramework curatorFramework) throws LockException { ZooKeeperHiveLock zLock = (ZooKeeperHiveLock)hiveLock; HiveLockObject obj = zLock.getHiveLockObject(); String name = getLastObjectName(parent, obj); try { - zkpClient.delete(zLock.getPath(), -1); + curatorFramework.delete().forPath(zLock.getPath()); // Delete the parent node if all the children have been deleted - List children = zkpClient.getChildren(name, false); + List children = curatorFramework.getChildren().forPath(name); if (children == null || children.isEmpty()) { - zkpClient.delete(name, -1); + curatorFramework.delete().forPath(name); } } catch (KeeperException.NoNodeException nne) { - //can happen in retrying deleting the zLock after exceptions like InterruptedException + //can happen in retrying deleting the zLock after exceptions like InterruptedException //or in a race condition where parent has already been deleted by other process when it //is to be deleted. Both cases should not raise error LOG.debug("Node " + zLock.getPath() + " or its parent has already been deleted."); @@ -480,7 +455,7 @@ //can happen in a race condition where another process adds a zLock under this parent //just before it is about to be deleted. It should not be a problem since this parent //can eventually be deleted by the process which hold its last child zLock - LOG.debug("Node " + name + " to be deleted is not empty."); + LOG.debug("Node " + name + " to be deleted is not empty."); } catch (Exception e) { //exceptions including InterruptException and other KeeperException LOG.error("Failed to release ZooKeeper lock: ", e); @@ -490,19 +465,14 @@ /* Release all locks - including PERSISTENT locks */ public static void releaseAllLocks(HiveConf conf) throws Exception { - ZooKeeper zkpClient = null; try { - int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); - String quorumServers = ZooKeeperHiveHelper.getQuorumServers(conf); - Watcher dummyWatcher = new ZooKeeperHiveHelper.DummyWatcher(); - zkpClient = new ZooKeeper(quorumServers, sessionTimeout, dummyWatcher); String parent = conf.getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_NAMESPACE); - List locks = getLocks(conf, zkpClient, null, parent, false, false); + List locks = getLocks(conf, null, parent, false, false); Exception lastExceptionGot = null; if (locks != null) { for (HiveLock lock : locks) { try { - unlockPrimitive(conf, zkpClient, lock, parent); + unlockPrimitive(lock, parent, curatorFramework); } catch (Exception e) { lastExceptionGot = e; } @@ -516,24 +486,19 @@ } catch (Exception e) { LOG.error("Failed to release all locks: ", e); throw new Exception(ErrorMsg.ZOOKEEPER_CLIENT_COULD_NOT_BE_INITIALIZED.getMsg()); - } finally { - if (zkpClient != null) { - zkpClient.close(); - zkpClient = null; - } } } /* Get all locks */ public List getLocks(boolean verifyTablePartition, boolean fetchData) throws LockException { - return getLocks(ctx.getConf(), zooKeeper, null, parent, verifyTablePartition, fetchData); + return getLocks(ctx.getConf(), null, parent, verifyTablePartition, fetchData); } /* Get all locks for a particular object */ public List getLocks(HiveLockObject key, boolean verifyTablePartitions, boolean fetchData) throws LockException { - return getLocks(ctx.getConf(), zooKeeper, key, parent, verifyTablePartitions, fetchData); + return getLocks(ctx.getConf(), key, parent, verifyTablePartitions, fetchData); } /** @@ -541,7 +506,7 @@ * @param zkpClient The ZooKeeper client * @param key The object to be compared against - if key is null, then get all locks **/ - private static List getLocks(HiveConf conf, ZooKeeper zkpClient, + private static List getLocks(HiveConf conf, HiveLockObject key, String parent, boolean verifyTablePartition, boolean fetchData) throws LockException { List locks = new ArrayList(); @@ -552,12 +517,12 @@ try { if (key != null) { commonParent = "/" + parent + "/" + key.getName(); - children = zkpClient.getChildren(commonParent, false); + children = curatorFramework.getChildren().forPath(commonParent); recurse = false; } else { commonParent = "/" + parent; - children = zkpClient.getChildren(commonParent, false); + children = curatorFramework.getChildren().forPath(commonParent); } } catch (Exception e) { // no locks present @@ -579,7 +544,7 @@ if (recurse) { try { - children = zkpClient.getChildren(curChild, false); + children = curatorFramework.getChildren().forPath(curChild); for (String child : children) { childn.add(curChild + "/" + child); } @@ -588,7 +553,7 @@ } } - HiveLockMode mode = getLockMode(conf, curChild); + HiveLockMode mode = getLockMode(curChild); if (mode == null) { continue; } @@ -605,8 +570,7 @@ if (fetchData) { try { - data = new HiveLockObjectData(new String(zkpClient.getData(curChild, - new ZooKeeperHiveHelper.DummyWatcher(), null))); + data = new HiveLockObjectData(new String(curatorFramework.getData().watched().forPath(curChild))); data.setClientIp(clientIp); } catch (Exception e) { LOG.error("Error in getting data for " + curChild, e); @@ -623,12 +587,7 @@ /** Remove all redundant nodes **/ private void removeAllRedundantNodes() { try { - renewZookeeperInstance(sessionTimeout, quorumServers); checkRedundantNode("/" + parent); - if (zooKeeper != null) { - zooKeeper.close(); - zooKeeper = null; - } } catch (Exception e) { LOG.warn("Exception while removing all redundant nodes", e); } @@ -637,19 +596,19 @@ private void checkRedundantNode(String node) { try { // Nothing to do if it is a lock mode - if (getLockMode(ctx.getConf(), node) != null) { + if (getLockMode(node) != null) { return; } - List children = zooKeeper.getChildren(node, false); + List children = curatorFramework.getChildren().forPath(node); for (String child : children) { checkRedundantNode(node + "/" + child); } - children = zooKeeper.getChildren(node, false); + children = curatorFramework.getChildren().forPath(node); if ((children == null) || (children.isEmpty())) { - zooKeeper.delete(node, -1); + curatorFramework.delete().forPath(node); } } catch (Exception e) { LOG.warn("Error in checkRedundantNode for node " + node, e); @@ -658,13 +617,8 @@ /* Release all transient locks, by simply closing the client */ public void close() throws LockException { - try { + try { - if (zooKeeper != null) { - zooKeeper.close(); - zooKeeper = null; - } - if (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES)) { removeAllRedundantNodes(); } @@ -750,7 +704,7 @@ private static Pattern exMode = Pattern.compile("^.*-(EXCLUSIVE)-([0-9]+)$"); /* Get the mode of the lock encoded in the path */ - private static HiveLockMode getLockMode(HiveConf conf, String path) { + private static HiveLockMode getLockMode(String path) { Matcher shMatcher = shMode.matcher(path); Matcher exMatcher = exMode.matcher(path); @@ -768,15 +722,6 @@ @Override public void prepareRetry() throws LockException { - try { - if (zooKeeper != null && zooKeeper.getState() == ZooKeeper.States.CLOSED) { - // Reconnect if the connection is closed. - zooKeeper = null; - } - renewZookeeperInstance(sessionTimeout, quorumServers); - } catch (Exception e) { - throw new LockException(e); - } } } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -29,6 +29,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,6 +37,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -48,6 +50,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.common.ObjectPair; @@ -1352,7 +1355,7 @@ } if (replace) { - Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf(), + Hive.replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), isSrcLocal); } else { FileSystem fs = tbl.getDataLocation().getFileSystem(conf); @@ -1411,7 +1414,7 @@ } /* dfs. */ - FileStatus[] children = fSys.listStatus(fSta.getPath()); + FileStatus[] children = fSys.listStatus(fSta.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); if (children != null) { for (FileStatus child : children) { walkDirTree(child, fSys, skewedColValueLocationMaps, newPartPath, skewedInfo); @@ -2187,7 +2190,7 @@ boolean grantOption) throws HiveException { try { return getMSC().grant_role(roleName, userName, principalType, grantor, - grantorType, grantOption); + grantorType, grantOption); } catch (Exception e) { throw new HiveException(e); } @@ -2282,13 +2285,7 @@ for (FileStatus src : srcs) { FileStatus[] items; if (src.isDir()) { - items = srcFs.listStatus(src.getPath(), new PathFilter() { - @Override - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - }); + items = srcFs.listStatus(src.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(items); } else { items = new FileStatus[] {src}; @@ -2308,9 +2305,10 @@ } if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES) && + !HiveConf.getVar(conf, HiveConf.ConfVars.STAGINGDIR).equals(itemSource.getName()) && item.isDir()) { throw new HiveException("checkPaths: " + src.getPath() - + " has nested directory" + itemSource); + + " has nested directory " + itemSource); } // Strip off the file type, if any so we don't make: // 000000_0.gz -> 000000_0.gz_copy_1 @@ -2361,11 +2359,54 @@ return false; } + private static boolean isSubDir(Path srcf, Path destf, FileSystem fs, boolean isSrcLocal){ + if (srcf == null) { + LOG.debug("The source path is null for isSubDir method."); + return false; + } + + String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, fs); + String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, fs); + + boolean isInTest = Boolean.valueOf(HiveConf.getBoolVar(fs.getConf(), ConfVars.HIVE_IN_TEST)); + // In the automation, the data warehouse is the local file system based. + LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2); + if (isInTest) { + return fullF1.startsWith(fullF2); + } + + // schema is diff, return false + String schemaSrcf = srcf.toUri().getScheme(); + String schemaDestf = destf.toUri().getScheme(); + + // if the schemaDestf is null, it means the destination is not in the local file system + if (schemaDestf == null && isSrcLocal) { + LOG.debug("The source file is in the local while the dest not."); + return false; + } + + // If both schema information are provided, they should be the same. + if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) { + LOG.debug("The source path's schema is " + schemaSrcf + + " and the destination path's schema is " + schemaDestf + "."); + return false; + } + + LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2); + return fullF1.startsWith(fullF2); + } + + private static String getQualifiedPathWithoutSchemeAndAuthority(Path srcf, FileSystem fs) { + Path currentWorkingDir = fs.getWorkingDirectory(); + Path path = srcf.makeQualified(srcf.toUri(), currentWorkingDir); + return Path.getPathWithoutSchemeAndAuthority(path).toString(); + } + //it is assumed that parent directory of the destf should already exist when this //method is called. when the replace value is true, this method works a little different //from mv command if the destf is a directory, it replaces the destf instead of moving under //the destf. in this case, the replaced destf still preserves the original destf's permission - public static boolean renameFile(HiveConf conf, Path srcf, Path destf, + public static boolean moveFile(HiveConf conf, Path srcf, Path destf, FileSystem fs, boolean replace, boolean isSrcLocal) throws HiveException { boolean success = false; @@ -2374,17 +2415,26 @@ HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); HadoopShims shims = ShimLoader.getHadoopShims(); HadoopShims.HdfsFileStatus destStatus = null; + HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); + // If source path is a subdirectory of the destination path: + // ex: INSERT OVERWRITE DIRECTORY 'target/warehouse/dest4.out' SELECT src.value WHERE src.key >= 300; + // where the staging directory is a subdirectory of the destination directory + // (1) Do not delete the dest dir before doing the move operation. + // (2) It is assumed that subdir and dir are in same encryption zone. + // (3) Move individual files from scr dir to dest dir. + boolean destIsSubDir = isSubDir(srcf, destf, fs, isSrcLocal); try { if (inheritPerms || replace) { try{ - destStatus = shims.getFullFileStatus(conf, fs, destf); + destStatus = shims.getFullFileStatus(conf, fs, destf.getParent()); //if destf is an existing directory: //if replace is true, delete followed by rename(mv) is equivalent to replace //if replace is false, rename (mv) actually move the src under dest dir //if destf is an existing file, rename is actually a replace, and do not need // to delete the file first - if (replace && destStatus.getFileStatus().isDir()) { + if (replace && !destIsSubDir) { + LOG.debug("The path " + destf.toString() + " is deleted"); fs.delete(destf, true); } } catch (FileNotFoundException ignore) { @@ -2396,14 +2446,39 @@ } if (!isSrcLocal) { // For NOT local src file, rename the file - success = fs.rename(srcf, destf); + if (hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf)) + && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf)) + { + LOG.info("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different."); + success = FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, + true, // delete source + replace, // overwrite destination + conf); + } else { + if (destIsSubDir) { + FileStatus[] srcs = fs.listStatus(srcf, FileUtils.HIDDEN_FILES_PATH_FILTER); + for (FileStatus status : srcs) { + success = FileUtils.copy(srcf.getFileSystem(conf), status.getPath(), destf.getFileSystem(conf), destf, + true, // delete source + replace, // overwrite destination + conf); + + if (!success) { + throw new HiveException("Unable to move source " + status.getPath() + " to destination " + destf); + } + } + } else { + success = fs.rename(srcf, destf); + } + } } else { // For local src file, copy to hdfs fs.copyFromLocalFile(srcf, destf); success = true; } - LOG.info((replace ? "Replacing src:" : "Renaming src:") + srcf.toString() - + ";dest: " + destf.toString() + ";Status:" + success); + + LOG.info((replace ? "Replacing src:" : "Renaming src: ") + srcf.toString() + + ", dest: " + destf.toString() + ", Status:" + success); } catch (IOException ioe) { throw new HiveException("Unable to move source " + srcf + " to destination " + destf, ioe); } @@ -2470,7 +2545,7 @@ try { for (List sdpairs : result) { for (Path[] sdpair : sdpairs) { - if (!renameFile(conf, sdpair[0], sdpair[1], fs, false, isSrcLocal)) { + if (!moveFile(conf, sdpair[0], sdpair[1], fs, false, isSrcLocal)) { throw new IOException("Cannot move " + sdpair[0] + " to " + sdpair[1]); } @@ -2563,6 +2638,7 @@ * srcf, destf, and tmppath should resident in the same DFS, but the oldPath can be in a * different DFS. * + * @param tablePath path of the table. Used to identify permission inheritance. * @param srcf * Source directory to be renamed to tmppath. It should be a * leaf directory where the final data files reside. However it @@ -2570,13 +2646,15 @@ * @param destf * The directory where the final data needs to go * @param oldPath - * The directory where the old data location, need to be cleaned up. + * The directory where the old data location, need to be cleaned up. Most of time, will be the same + * as destf, unless its across FileSystem boundaries. * @param isSrcLocal * If the source directory is LOCAL */ - static protected void replaceFiles(Path srcf, Path destf, Path oldPath, - HiveConf conf, boolean isSrcLocal) throws HiveException { + protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf, + boolean isSrcLocal) throws HiveException { try { + FileSystem destFs = destf.getFileSystem(conf); boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); @@ -2597,15 +2675,24 @@ List> result = checkPaths(conf, destFs, srcs, srcFs, destf, true); + HadoopShims shims = ShimLoader.getHadoopShims(); if (oldPath != null) { try { FileSystem fs2 = oldPath.getFileSystem(conf); if (fs2.exists(oldPath)) { - FileUtils.trashFilesUnderDir(fs2, oldPath, conf); + // Do not delete oldPath if: + // - destf is subdir of oldPath + //if ( !(fs2.equals(destf.getFileSystem(conf)) && FileUtils.isSubDir(oldPath, destf, fs2))) + if (FileUtils.isSubDir(oldPath, destf, fs2)) { + FileUtils.trashFilesUnderDir(fs2, oldPath, conf); + } + if (inheritPerms) { + inheritFromTable(tablePath, destf, conf, destFs); + } } } catch (Exception e) { //swallow the exception - LOG.warn("Directory " + oldPath.toString() + " canot be removed:" + StringUtils.stringifyException(e)); + LOG.warn("Directory " + oldPath.toString() + " cannot be removed: " + e, e); } } @@ -2619,15 +2706,30 @@ LOG.warn("Error creating directory " + destf.toString()); } if (inheritPerms && success) { - destFs.setPermission(destfp, destFs.getFileStatus(destfp.getParent()).getPermission()); + inheritFromTable(tablePath, destfp, conf, destFs); } } - boolean b = renameFile(conf, srcs[0].getPath(), destf, destFs, true, - isSrcLocal); - if (!b) { - throw new HiveException("Unable to move results from " + srcs[0].getPath() - + " to destination directory: " + destf); + // Copy/move each file under the source directory to avoid to delete the destination + // directory if it is the root of an HDFS encryption zone. + for (List sdpairs : result) { + for (Path[] sdpair : sdpairs) { + Path destParent = sdpair[1].getParent(); + FileSystem destParentFs = destParent.getFileSystem(conf); + if (!destParentFs.isDirectory(destParent)) { + boolean success = destFs.mkdirs(destParent); + if (!success) { + LOG.warn("Error creating directory " + destParent); + } + if (inheritPerms && success) { + inheritFromTable(tablePath, destParent, conf, destFs); + } + } + if (!moveFile(conf, sdpair[0], sdpair[1], destFs, true, isSrcLocal)) { + throw new IOException("Unable to move file/directory from " + sdpair[0] + + " to " + sdpair[1]); + } + } } } else { // srcf is a file or pattern containing wildcards if (!destFs.exists(destf)) { @@ -2636,13 +2738,13 @@ LOG.warn("Error creating directory " + destf.toString()); } if (inheritPerms && success) { - destFs.setPermission(destf, destFs.getFileStatus(destf.getParent()).getPermission()); + inheritFromTable(tablePath, destf, conf, destFs); } } // srcs must be a list of files -- ensured by LoadSemanticAnalyzer for (List sdpairs : result) { for (Path[] sdpair : sdpairs) { - if (!renameFile(conf, sdpair[0], sdpair[1], destFs, true, + if (!moveFile(conf, sdpair[0], sdpair[1], destFs, true, isSrcLocal)) { throw new IOException("Error moving: " + sdpair[0] + " into: " + sdpair[1]); } @@ -2654,6 +2756,38 @@ } } + /** + * This method sets all paths from tablePath to destf (including destf) to have same permission as tablePath. + * @param tablePath path of table + * @param destf path of table-subdir. + * @param conf + * @param fs + */ + private static void inheritFromTable(Path tablePath, Path destf, HiveConf conf, FileSystem fs) { + if (!FileUtils.isSubDir(destf, tablePath, fs)) { + //partition may not be under the parent. + return; + } + HadoopShims shims = ShimLoader.getHadoopShims(); + //Calculate all the paths from the table dir, to destf + //At end of this loop, currPath is table dir, and pathsToSet contain list of all those paths. + Path currPath = destf; + List pathsToSet = new LinkedList(); + while (!currPath.equals(tablePath)) { + pathsToSet.add(currPath); + currPath = currPath.getParent(); + } + + try { + HadoopShims.HdfsFileStatus fullFileStatus = shims.getFullFileStatus(conf, fs, currPath); + for (Path pathToSet : pathsToSet) { + shims.setFullFileStatus(conf, fullFileStatus, fs, pathToSet); + } + } catch (Exception e) { + LOG.warn("Error setting permissions or group of " + destf, e); + } + } + public static boolean isHadoop1() { return ShimLoader.getMajorVersion().startsWith("0.20"); } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; @@ -147,7 +148,7 @@ for (Path dbPath : dbPaths) { FileSystem fs = dbPath.getFileSystem(conf); - FileStatus[] statuses = fs.listStatus(dbPath); + FileStatus[] statuses = fs.listStatus(dbPath, FileUtils.HIDDEN_FILES_PATH_FILTER); for (FileStatus status : statuses) { if (status.isDir() && !tableNames.contains(status.getPath().getName())) { @@ -362,7 +363,7 @@ private void getAllLeafDirs(Path basePath, Set allDirs, FileSystem fs) throws IOException { - FileStatus[] statuses = fs.listStatus(basePath); + FileStatus[] statuses = fs.listStatus(basePath, FileUtils.HIDDEN_FILES_PATH_FILTER); boolean directoryFound=false; for (FileStatus status : statuses) { Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (working copy) @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.ProtectMode; import org.apache.hadoop.hive.metastore.Warehouse; @@ -336,7 +337,7 @@ * partition String pathPattern = this.partPath.toString() + "/*"; try { * FileSystem fs = FileSystem.get(this.table.getDataLocation(), * Hive.get().getConf()); FileStatus srcs[] = fs.globStatus(new - * Path(pathPattern)); numBuckets = srcs.length; } catch (Exception e) { + * Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); numBuckets = srcs.length; } catch (Exception e) { * throw new RuntimeException("Cannot get bucket count for table " + * this.table.getName(), e); } } return numBuckets; */ @@ -372,7 +373,7 @@ pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); - FileStatus srcs[] = fs.globStatus(new Path(pathPattern)); + FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.ProtectMode; @@ -636,7 +637,7 @@ protected void replaceFiles(Path srcf, boolean isSrcLocal) throws HiveException { Path tableDest = getPath(); - Hive.replaceFiles(srcf, tableDest, tableDest, Hive.get().getConf(), + Hive.replaceFiles(tableDest, srcf, tableDest, tableDest, Hive.get().getConf(), isSrcLocal); } @@ -951,7 +952,7 @@ pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); - FileStatus srcs[] = fs.globStatus(new Path(pathPattern)); + FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -47,7 +48,6 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.QB; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TableAccessAnalyzer; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -81,7 +81,7 @@ List fileNames = new ArrayList(); try { FileSystem fs = location.getFileSystem(pGraphContext.getConf()); - FileStatus[] files = fs.listStatus(new Path(location.toString())); + FileStatus[] files = fs.listStatus(new Path(location.toString()), FileUtils.HIDDEN_FILES_PATH_FILTER); if (files != null) { for (FileStatus file : files) { fileNames.add(file.getPath().toString()); @@ -132,23 +132,21 @@ protected boolean canConvertMapJoinToBucketMapJoin( MapJoinOperator mapJoinOp, - ParseContext pGraphContext, BucketJoinProcCtx context) throws SemanticException { - QBJoinTree joinCtx = pGraphContext.getMapJoinContext().get(mapJoinOp); - if (joinCtx == null) { + if (!this.pGraphContext.getMapJoinOps().contains(mapJoinOp)) { return false; } List joinAliases = new ArrayList(); - String[] srcs = joinCtx.getBaseSrc(); - String[] left = joinCtx.getLeftAliases(); - List mapAlias = joinCtx.getMapAliases(); + String[] srcs = mapJoinOp.getConf().getBaseSrc(); + String[] left = mapJoinOp.getConf().getLeftAliases(); + List mapAlias = mapJoinOp.getConf().getMapAliases(); String baseBigAlias = null; for (String s : left) { if (s != null) { - String subQueryAlias = QB.getAppendedAliasFromId(joinCtx.getId(), s); + String subQueryAlias = QB.getAppendedAliasFromId(mapJoinOp.getConf().getId(), s); if (!joinAliases.contains(subQueryAlias)) { joinAliases.add(subQueryAlias); if (!mapAlias.contains(s)) { @@ -160,7 +158,7 @@ for (String s : srcs) { if (s != null) { - String subQueryAlias = QB.getAppendedAliasFromId(joinCtx.getId(), s); + String subQueryAlias = QB.getAppendedAliasFromId(mapJoinOp.getConf().getId(), s); if (!joinAliases.contains(subQueryAlias)) { joinAliases.add(subQueryAlias); if (!mapAlias.contains(s)) { @@ -173,9 +171,8 @@ Map> keysMap = mapJoinOp.getConf().getKeys(); return checkConvertBucketMapJoin( - pGraphContext, context, - joinCtx, + mapJoinOp.getConf().getAliasToOpInfo(), keysMap, baseBigAlias, joinAliases); @@ -190,9 +187,8 @@ * d. The number of buckets in the big table can be divided by no of buckets in small tables. */ protected boolean checkConvertBucketMapJoin( - ParseContext pGraphContext, BucketJoinProcCtx context, - QBJoinTree joinCtx, + Map> aliasToOpInfo, Map> keysMap, String baseBigAlias, List joinAliases) throws SemanticException { @@ -203,7 +199,6 @@ new LinkedHashMap>>(); HashMap> topOps = pGraphContext.getTopOps(); - Map topToTable = pGraphContext.getTopToTable(); HashMap aliasToNewAliasMap = new HashMap(); @@ -218,7 +213,7 @@ boolean bigTablePartitioned = true; for (int index = 0; index < joinAliases.size(); index++) { String alias = joinAliases.get(index); - Operator topOp = joinCtx.getAliasToOpInfo().get(alias); + Operator topOp = aliasToOpInfo.get(alias); // The alias may not be present in case of a sub-query if (topOp == null) { return false; @@ -270,7 +265,7 @@ joinKeyOrder = new Integer[keys.size()]; } - Table tbl = topToTable.get(tso); + Table tbl = tso.getConf().getTableMetadata(); if (tbl.isPartitioned()) { PrunedPartitionList prunedParts = pGraphContext.getPrunedPartitions(alias, tso); List partitions = prunedParts.getNotDeniedPartns(); @@ -459,7 +454,7 @@ } // convert partition to partition spec string - private static Map> convert(Map> mapping) { + private Map> convert(Map> mapping) { Map> converted = new HashMap>(); for (Map.Entry> entry : mapping.entrySet()) { converted.put(entry.getKey().getName(), entry.getValue()); @@ -488,7 +483,7 @@ } // called for each partition of big table and populates mapping for each file in the partition - private static void fillMappingBigTableBucketFileNameToSmallTableBucketFileNames( + private void fillMappingBigTableBucketFileNameToSmallTableBucketFileNames( List smallTblBucketNums, List> smallTblFilesList, Map> bigTableBucketFileNameToSmallTableBucketFileNames, Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java (working copy) @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.QB; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TableAccessAnalyzer; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -97,17 +96,17 @@ return false; } - boolean tableEligibleForBucketedSortMergeJoin = true; - QBJoinTree joinCxt = this.pGraphContext.getMapJoinContext() - .get(mapJoinOp); - if (joinCxt == null) { + if (!this.pGraphContext.getMapJoinOps().contains(mapJoinOp)) { return false; } - String[] srcs = joinCxt.getBaseSrc(); + + String[] srcs = mapJoinOp.getConf().getBaseSrc(); for (int srcPos = 0; srcPos < srcs.length; srcPos++) { - srcs[srcPos] = QB.getAppendedAliasFromId(joinCxt.getId(), srcs[srcPos]); + srcs[srcPos] = QB.getAppendedAliasFromId(mapJoinOp.getConf().getId(), srcs[srcPos]); } + boolean tableEligibleForBucketedSortMergeJoin = true; + // All the tables/partitions columns should be sorted in the same order // For example, if tables A and B are being joined on columns c1, c2 and c3 // which are the sorted and bucketed columns. The join would work, as long @@ -117,9 +116,8 @@ for (int pos = 0; pos < srcs.length; pos++) { tableEligibleForBucketedSortMergeJoin = tableEligibleForBucketedSortMergeJoin && isEligibleForBucketSortMergeJoin(smbJoinContext, - pGraphContext, mapJoinOp.getConf().getKeys().get((byte) pos), - joinCxt, + mapJoinOp.getConf().getAliasToOpInfo(), srcs, pos, sortColumnsFirstTable); @@ -141,8 +139,7 @@ // Convert the bucket map-join operator to a sort-merge map join operator protected SMBMapJoinOperator convertBucketMapJoinToSMBJoin(MapJoinOperator mapJoinOp, - SortBucketJoinProcCtx smbJoinContext, - ParseContext parseContext) { + SortBucketJoinProcCtx smbJoinContext) { String[] srcs = smbJoinContext.getSrcs(); SMBMapJoinOperator smbJop = new SMBMapJoinOperator(mapJoinOp); @@ -219,11 +216,14 @@ child.getParentOperators().remove(index); child.getParentOperators().add(index, smbJop); } - parseContext.getSmbMapJoinContext().put(smbJop, - parseContext.getMapJoinContext().get(mapJoinOp)); - parseContext.getMapJoinContext().remove(mapJoinOp); - parseContext.getOpParseCtx().put(smbJop, parseContext.getOpParseCtx().get(mapJoinOp)); + // Data structures coming from QBJoinTree + smbJop.getConf().setQBJoinTreeProps(mapJoinOp.getConf()); + // + pGraphContext.getSmbMapJoinOps().add(smbJop); + pGraphContext.getMapJoinOps().remove(mapJoinOp); + pGraphContext.getOpParseCtx().put(smbJop, pGraphContext.getOpParseCtx().get(mapJoinOp)); + return smbJop; } @@ -242,15 +242,12 @@ */ private boolean isEligibleForBucketSortMergeJoin( SortBucketJoinProcCtx smbJoinContext, - ParseContext pctx, List keys, - QBJoinTree joinTree, + Map> aliasToOpInfo, String[] aliases, int pos, List sortColumnsFirstTable) throws SemanticException { String alias = aliases[pos]; - Map topToTable = this.pGraphContext - .getTopToTable(); /* * Consider a query like: @@ -266,7 +263,7 @@ * table. If the object being map-joined is a base table, then aliasToOpInfo * contains the TableScanOperator, and TableAccessAnalyzer is a no-op. */ - Operator topOp = joinTree.getAliasToOpInfo().get(alias); + Operator topOp = aliasToOpInfo.get(alias); if (topOp == null) { return false; } @@ -313,7 +310,7 @@ return false; } - Table tbl = topToTable.get(tso); + Table tbl = tso.getConf().getTableMetadata(); if (tbl.isPartitioned()) { PrunedPartitionList prunedParts = pGraphContext.getPrunedPartitions(alias, tso); List partitions = prunedParts.getNotDeniedPartns(); @@ -386,16 +383,14 @@ // It is already verified that the join can be converted to a bucket map join protected boolean checkConvertJoinToSMBJoin( JoinOperator joinOperator, - SortBucketJoinProcCtx smbJoinContext, - ParseContext pGraphContext) throws SemanticException { + SortBucketJoinProcCtx smbJoinContext) throws SemanticException { - QBJoinTree joinCtx = pGraphContext.getJoinContext().get(joinOperator); - - if (joinCtx == null) { + if (!this.pGraphContext.getJoinOps().contains(joinOperator)) { return false; } - String[] srcs = joinCtx.getBaseSrc(); + String[] srcs = joinOperator.getConf().getBaseSrc(); + // All the tables/partitions columns should be sorted in the same order // For example, if tables A and B are being joined on columns c1, c2 and c3 // which are the sorted and bucketed columns. The join would work, as long @@ -404,9 +399,8 @@ for (int pos = 0; pos < srcs.length; pos++) { if (!isEligibleForBucketSortMergeJoin(smbJoinContext, - pGraphContext, smbJoinContext.getKeyExprMap().get((byte) pos), - joinCtx, + joinOperator.getConf().getAliasToOpInfo(), srcs, pos, sortColumnsFirstTable)) { @@ -421,12 +415,10 @@ // Can the join operator be converted to a sort-merge join operator ? protected boolean canConvertJoinToSMBJoin( JoinOperator joinOperator, - SortBucketJoinProcCtx smbJoinContext, - ParseContext pGraphContext) throws SemanticException { + SortBucketJoinProcCtx smbJoinContext) throws SemanticException { boolean canConvert = canConvertJoinToBucketMapJoin( joinOperator, - pGraphContext, smbJoinContext ); @@ -434,13 +426,12 @@ return false; } - return checkConvertJoinToSMBJoin(joinOperator, smbJoinContext, pGraphContext); + return checkConvertJoinToSMBJoin(joinOperator, smbJoinContext); } // Can the join operator be converted to a bucket map-merge join operator ? protected boolean canConvertJoinToBucketMapJoin( JoinOperator joinOp, - ParseContext pGraphContext, SortBucketJoinProcCtx context) throws SemanticException { // This has already been inspected and rejected @@ -448,8 +439,7 @@ return false; } - QBJoinTree joinCtx = pGraphContext.getJoinContext().get(joinOp); - if (joinCtx == null) { + if (!this.pGraphContext.getJoinOps().contains(joinOp)) { return false; } @@ -482,8 +472,9 @@ context.setBigTablePosition(bigTablePosition); String joinAlias = bigTablePosition == 0 ? - joinCtx.getLeftAlias() : joinCtx.getRightAliases()[bigTablePosition - 1]; - joinAlias = QB.getAppendedAliasFromId(joinCtx.getId(), joinAlias); + joinOp.getConf().getLeftAlias() : + joinOp.getConf().getRightAliases()[bigTablePosition - 1]; + joinAlias = QB.getAppendedAliasFromId(joinOp.getConf().getId(), joinAlias); Map> keyExprMap = new HashMap>(); List> parentOps = joinOp.getParentOperators(); @@ -497,10 +488,10 @@ context.setKeyExprMap(keyExprMap); // Make a deep copy of the aliases so that they are not changed in the context - String[] joinSrcs = joinCtx.getBaseSrc(); + String[] joinSrcs = joinOp.getConf().getBaseSrc(); String[] srcs = new String[joinSrcs.length]; for (int srcPos = 0; srcPos < joinSrcs.length; srcPos++) { - joinSrcs[srcPos] = QB.getAppendedAliasFromId(joinCtx.getId(), joinSrcs[srcPos]); + joinSrcs[srcPos] = QB.getAppendedAliasFromId(joinOp.getConf().getId(), joinSrcs[srcPos]); srcs[srcPos] = new String(joinSrcs[srcPos]); } @@ -508,9 +499,8 @@ // The candidate map-join was derived from the pluggable sort merge join big // table matcher. return checkConvertBucketMapJoin( - pGraphContext, context, - joinCtx, + joinOp.getConf().getAliasToOpInfo(), keyExprMap, joinAlias, Arrays.asList(srcs)); @@ -519,19 +509,23 @@ // Convert the join operator to a bucket map-join join operator protected MapJoinOperator convertJoinToBucketMapJoin( JoinOperator joinOp, - SortBucketJoinProcCtx joinContext, - ParseContext parseContext) throws SemanticException { + SortBucketJoinProcCtx joinContext) throws SemanticException { MapJoinOperator mapJoinOp = new MapJoinProcessor().convertMapJoin( - parseContext.getConf(), - parseContext.getOpParseCtx(), + pGraphContext.getConf(), + pGraphContext.getOpParseCtx(), joinOp, - pGraphContext.getJoinContext().get(joinOp), + joinOp.getConf().isLeftInputJoin(), + joinOp.getConf().getBaseSrc(), + joinOp.getConf().getMapAliases(), joinContext.getBigTablePosition(), false, false); // Remove the join operator from the query join context - parseContext.getMapJoinContext().put(mapJoinOp, parseContext.getJoinContext().get(joinOp)); - parseContext.getJoinContext().remove(joinOp); + // Data structures coming from QBJoinTree + mapJoinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); + // + pGraphContext.getMapJoinOps().add(mapJoinOp); + pGraphContext.getJoinOps().remove(joinOp); convertMapJoinToBucketMapJoin(mapJoinOp, joinContext); return mapJoinOp; } @@ -539,11 +533,10 @@ // Convert the join operator to a sort-merge join operator protected void convertJoinToSMBJoin( JoinOperator joinOp, - SortBucketJoinProcCtx smbJoinContext, - ParseContext parseContext) throws SemanticException { - MapJoinOperator mapJoinOp = convertJoinToBucketMapJoin(joinOp, smbJoinContext, parseContext); + SortBucketJoinProcCtx smbJoinContext) throws SemanticException { + MapJoinOperator mapJoinOp = convertJoinToBucketMapJoin(joinOp, smbJoinContext); SMBMapJoinOperator smbMapJoinOp = - convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext, parseContext); + convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext); smbMapJoinOp.setConvertedAutomaticallySMBJoin(true); } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java (working copy) @@ -71,7 +71,7 @@ int numPartitions = 1; // in case the sizes match, preference is // given to the table with fewer partitions - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); long averageSize = 0; if (!table.isPartitioned()) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapjoinProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapjoinProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapjoinProc.java (working copy) @@ -45,7 +45,7 @@ // can the mapjoin present be converted to a bucketed mapjoin boolean convert = canConvertMapJoinToBucketMapJoin( - mapJoinOperator, pGraphContext, context); + mapJoinOperator, context); HiveConf conf = context.getConf(); // Throw an error if the user asked for bucketed mapjoin to be enforced and @@ -67,13 +67,13 @@ * and do the version if possible. */ public static void checkAndConvertBucketMapJoin(ParseContext pGraphContext, - MapJoinOperator mapJoinOp, QBJoinTree joinCtx, String baseBigAlias, + MapJoinOperator mapJoinOp, String baseBigAlias, List joinAliases) throws SemanticException { BucketJoinProcCtx ctx = new BucketJoinProcCtx(pGraphContext.getConf()); BucketMapjoinProc proc = new BucketMapjoinProc(pGraphContext); Map> keysMap = mapJoinOp.getConf().getKeys(); - if (proc.checkConvertBucketMapJoin(pGraphContext, ctx, - joinCtx, keysMap, baseBigAlias, joinAliases)) { + if (proc.checkConvertBucketMapJoin(ctx, mapJoinOp.getConf().getAliasToOpInfo(), + keysMap, baseBigAlias, joinAliases)) { proc.convertMapJoinToBucketMapJoin(mapJoinOp, ctx); } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java (working copy) @@ -390,7 +390,7 @@ } } - Table destTable = pGraphContext.getFsopToTable().get(fsOp); + Table destTable = fsOp.getConf().getTable(); if (destTable == null) { return null; } @@ -465,7 +465,7 @@ if (op instanceof TableScanOperator) { assert !useBucketSortPositions; TableScanOperator ts = (TableScanOperator) op; - Table srcTable = pGraphContext.getTopToTable().get(ts); + Table srcTable = ts.getConf().getTableMetadata(); // Find the positions of the bucketed columns in the table corresponding // to the select list. Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (working copy) @@ -141,6 +141,17 @@ colLists = Utilities.mergeUniqElems(colLists, param.getCols()); } } + int groupingSetPosition = conf.getGroupingSetPosition(); + if (groupingSetPosition >= 0) { + List cols = cppCtx.genColLists(op); + String groupingColumn = conf.getOutputColumnNames().get(groupingSetPosition); + if (!cols.contains(groupingColumn)) { + conf.getOutputColumnNames().remove(groupingSetPosition); + if (op.getSchema() != null) { + op.getSchema().getSignature().remove(groupingSetPosition); + } + } + } cppCtx.getPrunedColLists().put(op, colLists); return null; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (working copy) @@ -863,7 +863,7 @@ if (op.getChildOperators().size() == 1 && op.getChildOperators().get(0) instanceof JoinOperator) { JoinOperator joinOp = (JoinOperator) op.getChildOperators().get(0); - if (skipFolding(joinOp.getConf(), rsDesc.getTag())) { + if (skipFolding(joinOp.getConf())) { LOG.debug("Skip folding in outer join " + op); cppCtx.getOpToConstantExprs().put(op, new HashMap()); return null; @@ -889,22 +889,16 @@ rsDesc.setKeyCols(newKeyEpxrs); // partition columns - if (!rsDesc.getPartitionCols().isEmpty()) { - ArrayList newPartExprs = new ArrayList(); - for (ExprNodeDesc desc : rsDesc.getPartitionCols()) { - ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false); - if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeNullDesc) { - continue; - } - newPartExprs.add(expr); + ArrayList newPartExprs = new ArrayList(); + for (ExprNodeDesc desc : rsDesc.getPartitionCols()) { + ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false); + if (expr != desc && desc instanceof ExprNodeColumnDesc + && expr instanceof ExprNodeConstantDesc) { + ((ExprNodeConstantDesc) expr).setFoldedFromCol(((ExprNodeColumnDesc) desc).getColumn()); } - if (newPartExprs.isEmpty()) { - // If all partition columns are removed because of constant, insert an extra column to avoid - // random partitioning. - newPartExprs.add(new ExprNodeConstantDesc("")); - } - rsDesc.setPartitionCols(newPartExprs); + newPartExprs.add(expr); } + rsDesc.setPartitionCols(newPartExprs); // value columns ArrayList newValExprs = new ArrayList(); @@ -916,28 +910,19 @@ return null; } - private boolean skipFolding(JoinDesc joinDesc, int tag) { - JoinCondDesc[] conds = joinDesc.getConds(); - int i; - for (i = conds.length - 1; i >= 0; i--) { - if (conds[i].getType() == JoinDesc.INNER_JOIN) { - if (tag == i + 1) - return false; - } else if (conds[i].getType() == JoinDesc.FULL_OUTER_JOIN) { - return true; - } else if (conds[i].getType() == JoinDesc.RIGHT_OUTER_JOIN) { - if (tag == i + 1) - return false; - return true; - } else if (conds[i].getType() == JoinDesc.LEFT_OUTER_JOIN) { - if (tag == i + 1) - return true; + /** + * Skip folding constants if there is outer join in join tree. + * @param joinDesc + * @return true if to skip. + */ + private boolean skipFolding(JoinDesc joinDesc) { + for (JoinCondDesc cond : joinDesc.getConds()) { + if (cond.getType() == JoinDesc.INNER_JOIN || cond.getType() == JoinDesc.UNIQUE_JOIN) { + continue; } + return true; } - if (tag == 0) { - return false; - } - return true; + return false; } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (working copy) @@ -47,8 +47,8 @@ import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc; import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; -import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; @@ -231,13 +231,16 @@ ParseContext parseContext = context.parseContext; MapJoinDesc mapJoinDesc = null; if (adjustParentsChildren) { - mapJoinDesc = MapJoinProcessor.getMapJoinDesc(context.conf, parseContext.getOpParseCtx(), - joinOp, parseContext.getJoinContext().get(joinOp), mapJoinConversionPos, true); + mapJoinDesc = MapJoinProcessor.getMapJoinDesc(context.conf, parseContext.getOpParseCtx(), + joinOp, joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp.getConf().getMapAliases(), + mapJoinConversionPos, true); } else { JoinDesc joinDesc = joinOp.getConf(); // retain the original join desc in the map join. mapJoinDesc = - new MapJoinDesc(MapJoinProcessor.getKeys(parseContext.getJoinContext().get(joinOp), joinOp).getSecond(), + new MapJoinDesc( + MapJoinProcessor.getKeys(joinOp.getConf().isLeftInputJoin(), + joinOp.getConf().getBaseSrc(), joinOp).getSecond(), null, joinDesc.getExprs(), null, null, joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(), joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null); @@ -504,7 +507,38 @@ } public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext context, - int buckets) { + int buckets) throws SemanticException { + /* + * HIVE-9038: Join tests fail in tez when we have more than 1 join on the same key and there is + * an outer join down the join tree that requires filterTag. We disable this conversion to map + * join here now. We need to emulate the behavior of HashTableSinkOperator as in MR or create a + * new operation to be able to support this. This seems like a corner case enough to special + * case this for now. + */ + if (joinOp.getConf().getConds().length > 1) { + boolean hasOuter = false; + for (JoinCondDesc joinCondDesc : joinOp.getConf().getConds()) { + switch (joinCondDesc.getType()) { + case JoinDesc.INNER_JOIN: + case JoinDesc.LEFT_SEMI_JOIN: + case JoinDesc.UNIQUE_JOIN: + hasOuter = false; + break; + + case JoinDesc.FULL_OUTER_JOIN: + case JoinDesc.LEFT_OUTER_JOIN: + case JoinDesc.RIGHT_OUTER_JOIN: + hasOuter = true; + break; + + default: + throw new SemanticException("Unknown join type " + joinCondDesc.getType()); + } + } + if (hasOuter) { + return -1; + } + } Set bigTableCandidateSet = MapJoinProcessor.getBigTableCandidates(joinOp.getConf().getConds()); @@ -606,7 +640,8 @@ ParseContext parseContext = context.parseContext; MapJoinOperator mapJoinOp = MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, parseContext.getOpParseCtx(), joinOp, - parseContext.getJoinContext().get(joinOp), bigTablePosition, true); + joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp.getConf().getMapAliases(), + bigTablePosition, true); Operator parentBigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java (working copy) @@ -186,7 +186,7 @@ String column = extractColName(ctx.parent); if (ts != null && column != null) { - Table table = parseContext.getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); if (table != null && table.isPartitionKey(column)) { String alias = ts.getConf().getAlias(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (working copy) @@ -70,7 +70,7 @@ TableScanOperator op = (TableScanOperator) nd; GenMRProcContext ctx = (GenMRProcContext) opProcCtx; ParseContext parseCtx = ctx.getParseCtx(); - Class inputFormat = parseCtx.getTopToTable().get(op) + Class inputFormat = op.getConf().getTableMetadata() .getInputFormatClass(); Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy) @@ -18,7 +18,20 @@ package org.apache.hadoop.hive.ql.optimizer; -import com.google.common.collect.Interner; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -67,7 +80,6 @@ import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.QBParseInfo; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -102,19 +114,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.InputFormat; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; +import com.google.common.collect.Interner; /** * General utility common functions for the Processor to convert operator into @@ -521,12 +521,11 @@ // The table does not have any partitions if (aliasPartnDesc == null) { - aliasPartnDesc = new PartitionDesc(Utilities.getTableDesc(parseCtx - .getTopToTable().get(topOp)), null); - + aliasPartnDesc = new PartitionDesc(Utilities.getTableDesc(((TableScanOperator) topOp) + .getConf().getTableMetadata()), null); } - Map props = parseCtx.getTopToProps().get(topOp); + Map props = topOp.getConf().getOpProps(); if (props != null) { Properties target = aliasPartnDesc.getProperties(); if (target == null) { @@ -955,7 +954,7 @@ public static TableScanOperator createTemporaryTableScanOperator(RowSchema rowSchema) { TableScanOperator tableScanOp = - (TableScanOperator) OperatorFactory.get(new TableScanDesc(), rowSchema); + (TableScanOperator) OperatorFactory.get(new TableScanDesc(null), rowSchema); // Set needed columns for this dummy TableScanOperator List neededColumnIds = new ArrayList(); List neededColumnNames = new ArrayList(); @@ -1067,17 +1066,23 @@ if (needsTagging(cplan.getReduceWork())) { Operator reducerOp = cplan.getReduceWork().getReducer(); - QBJoinTree joinTree = null; + String id = null; if (reducerOp instanceof JoinOperator) { - joinTree = parseCtx.getJoinContext().get(reducerOp); + if (parseCtx.getJoinOps().contains(reducerOp)) { + id = ((JoinOperator)reducerOp).getConf().getId(); + } } else if (reducerOp instanceof MapJoinOperator) { - joinTree = parseCtx.getMapJoinContext().get(reducerOp); + if (parseCtx.getMapJoinOps().contains(reducerOp)) { + id = ((MapJoinOperator)reducerOp).getConf().getId(); + } } else if (reducerOp instanceof SMBMapJoinOperator) { - joinTree = parseCtx.getSmbMapJoinContext().get(reducerOp); + if (parseCtx.getSmbMapJoinOps().contains(reducerOp)) { + id = ((SMBMapJoinOperator)reducerOp).getConf().getId(); + } } - if (joinTree != null && joinTree.getId() != null) { - streamDesc = joinTree.getId() + ":$INTNAME"; + if (id != null) { + streamDesc = id + ":$INTNAME"; } else { streamDesc = "$INTNAME"; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java (working copy) @@ -62,10 +62,8 @@ GlobalLimitCtx globalLimitCtx = pctx.getGlobalLimitCtx(); Map opToPartPruner = pctx.getOpToPartPruner(); Map nameToSplitSample = pctx.getNameToSplitSample(); - Map topToTable = pctx.getTopToTable(); QB qb = pctx.getQB(); - HiveConf conf = pctx.getConf(); QBParseInfo qbParseInfo = qb.getParseInfo(); // determine the query qualifies reduce input size for LIMIT @@ -93,7 +91,7 @@ // query qualify for the optimization if (tempGlobalLimit != null && tempGlobalLimit != 0) { TableScanOperator ts = (TableScanOperator) topOps.values().toArray()[0]; - Table tab = topToTable.get(ts); + Table tab = ts.getConf().getTableMetadata(); if (!tab.isPartitioned()) { if (qbParseInfo.getDestToWhereExpr().isEmpty()) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java (working copy) @@ -304,7 +304,7 @@ // Create a mapping from the group by columns to the table columns Map tableColsMapping = new HashMap(); Set constantCols = new HashSet(); - Table table = pGraphContext.getTopToTable().get(currOp); + Table table = tableScanOp.getConf().getTableMetadata(); for (FieldSchema col : table.getAllCols()) { tableColsMapping.put(col.getName(), col.getName()); } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (working copy) @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.ql.Driver; @@ -123,22 +124,23 @@ Partition part) throws HiveException { LOG.info("checking index staleness..."); try { - FileSystem partFs = part.getDataLocation().getFileSystem(hive.getConf()); - FileStatus partFss = partFs.getFileStatus(part.getDataLocation()); - String ts = index.getParameters().get(part.getSpec().toString()); - if (ts == null) { + String indexTs = index.getParameters().get(part.getSpec().toString()); + if (indexTs == null) { return false; } - long indexTs = Long.parseLong(ts); - LOG.info(partFss.getModificationTime()); - LOG.info(ts); - if (partFss.getModificationTime() > indexTs) { - LOG.info("index is stale on the partitions that matched " + part.getSpec()); - return false; + + FileSystem partFs = part.getDataLocation().getFileSystem(hive.getConf()); + FileStatus[] parts = partFs.listStatus(part.getDataLocation(), FileUtils.HIDDEN_FILES_PATH_FILTER); + for (FileStatus status : parts) { + if (status.getModificationTime() > Long.parseLong(indexTs)) { + LOG.info("Index is stale on partition '" + part.getName() + + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + + "' is higher than index creation time (" + indexTs + ")."); + return false; + } } } catch (IOException e) { - LOG.info("failed to grab timestamp info"); - throw new HiveException(e); + throw new HiveException("Failed to grab timestamp information from partition '" + part.getName() + "': " + e.getMessage(), e); } return true; } @@ -156,22 +158,23 @@ for (Index index : indexes) { LOG.info("checking index staleness..."); try { - FileSystem srcFs = src.getPath().getFileSystem(hive.getConf()); - FileStatus srcFss= srcFs.getFileStatus(src.getPath()); - String ts = index.getParameters().get("base_timestamp"); - if (ts == null) { + String indexTs = index.getParameters().get("base_timestamp"); + if (indexTs == null) { return false; } - long indexTs = Long.parseLong(ts); - LOG.info(srcFss.getModificationTime()); - LOG.info(ts); - if (srcFss.getModificationTime() > indexTs) { - LOG.info("index is stale "); - return false; + + FileSystem srcFs = src.getPath().getFileSystem(hive.getConf()); + FileStatus[] srcs = srcFs.listStatus(src.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); + for (FileStatus status : srcs) { + if (status.getModificationTime() > Long.parseLong(indexTs)) { + LOG.info("Index is stale on table '" + src.getTableName() + + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + + "' is higher than index creation time (" + indexTs + ")."); + return false; + } } } catch (IOException e) { - LOG.info("failed to grab timestamp info"); - throw new HiveException(e); + throw new HiveException("Failed to grab timestamp information from table '" + src.getTableName() + "': " + e.getMessage(), e); } } return true; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java (working copy) @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -95,9 +94,9 @@ private Set getBigTables(ParseContext joinCtx) { Set bigTables = new HashSet(); - for (QBJoinTree qbJoin : joinCtx.getJoinContext().values()) { - if (qbJoin.getStreamAliases() != null) { - bigTables.addAll(qbJoin.getStreamAliases()); + for (JoinOperator joinOp : joinCtx.getJoinOps()) { + if (joinOp.getConf().getStreamAliases() != null) { + bigTables.addAll(joinOp.getConf().getStreamAliases()); } } @@ -155,7 +154,7 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { Set bigTables = getBigTables(pactx); - for (JoinOperator joinOp : pactx.getJoinContext().keySet()) { + for (JoinOperator joinOp : pactx.getJoinOps()) { reorder(joinOp, bigTables); } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (working copy) @@ -61,7 +61,6 @@ import org.apache.hadoop.hive.ql.parse.GenMapRedWalker; import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -96,18 +95,13 @@ // needs to be passed. Use the string defined below for that. private static final String MAPJOINKEY_FIELDPREFIX = "mapjoinkey"; - private ParseContext pGraphContext; - - /** - * empty constructor. - */ public MapJoinProcessor() { - pGraphContext = null; } @SuppressWarnings("nls") - private Operator - putOpInsertMap(Operator op, RowResolver rr) { + private static Operator putOpInsertMap ( + ParseContext pGraphContext, Operator op, + RowResolver rr) { OpParseContext ctx = new OpParseContext(rr); pGraphContext.getOpParseCtx().put(op, ctx); return op; @@ -232,10 +226,10 @@ throws SemanticException { LinkedHashMap, OpParseContext> opParseCtxMap = newWork.getMapWork().getOpParseCtxMap(); - QBJoinTree newJoinTree = newWork.getMapWork().getJoinTree(); // generate the map join operator; already checked the map join MapJoinOperator newMapJoinOp = new MapJoinProcessor().convertMapJoin(conf, opParseCtxMap, op, - newJoinTree, mapJoinPos, true, false); + newWork.getMapWork().isLeftInputJoin(), newWork.getMapWork().getBaseSrc(), newWork.getMapWork().getMapAliases(), + mapJoinPos, true, false); genLocalWorkForMapJoin(newWork, newMapJoinOp, mapJoinPos); } @@ -247,7 +241,9 @@ MapJoinProcessor.genMapJoinLocalWork(newWork, newMapJoinOp, mapJoinPos); // clean up the mapred work newWork.getMapWork().setOpParseCtxMap(null); - newWork.getMapWork().setJoinTree(null); + newWork.getMapWork().setLeftInputJoin(false); + newWork.getMapWork().setBaseSrc(null); + newWork.getMapWork().setMapAliases(null); } catch (Exception e) { e.printStackTrace(); @@ -307,9 +303,8 @@ */ public MapJoinOperator convertMapJoin(HiveConf conf, LinkedHashMap, OpParseContext> opParseCtxMap, - JoinOperator op, QBJoinTree joinTree, int mapJoinPos, boolean noCheckOuterJoin, - boolean validateMapJoinTree) - throws SemanticException { + JoinOperator op, boolean leftInputJoin, String[] baseSrc, List mapAliases, + int mapJoinPos, boolean noCheckOuterJoin, boolean validateMapJoinTree) throws SemanticException { // outer join cannot be performed on a table which is being cached JoinDesc desc = op.getConf(); @@ -324,8 +319,6 @@ // Walk over all the sources (which are guaranteed to be reduce sink // operators). // The join outputs a concatenation of all the inputs. - QBJoinTree leftSrc = joinTree.getJoinSrc(); - List> parentOps = op.getParentOperators(); List> newParentOps = new ArrayList>(); @@ -333,7 +326,7 @@ new ArrayList>(); // found a source which is not to be stored in memory - if (leftSrc != null) { + if (leftInputJoin) { // assert mapJoinPos == 0; Operator parentOp = parentOps.get(0); assert parentOp.getParentOperators().size() == 1; @@ -345,7 +338,7 @@ byte pos = 0; // Remove parent reduce-sink operators - for (String src : joinTree.getBaseSrc()) { + for (String src : baseSrc) { if (src != null) { Operator parentOp = parentOps.get(pos); assert parentOp.getParentOperators().size() == 1; @@ -360,7 +353,7 @@ // create the map-join operator MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf, opParseCtxMap, - op, joinTree, mapJoinPos, noCheckOuterJoin); + op, leftInputJoin, baseSrc, mapAliases, mapJoinPos, noCheckOuterJoin); // remove old parents @@ -384,11 +377,12 @@ public static MapJoinOperator convertJoinOpMapJoinOp(HiveConf hconf, LinkedHashMap, OpParseContext> opParseCtxMap, - JoinOperator op, QBJoinTree joinTree, int mapJoinPos, boolean noCheckOuterJoin) - throws SemanticException { + JoinOperator op, boolean leftInputJoin, String[] baseSrc, List mapAliases, + int mapJoinPos, boolean noCheckOuterJoin) throws SemanticException { MapJoinDesc mapJoinDescriptor = - getMapJoinDesc(hconf, opParseCtxMap, op, joinTree, mapJoinPos, noCheckOuterJoin); + getMapJoinDesc(hconf, opParseCtxMap, op, leftInputJoin, baseSrc, mapAliases, + mapJoinPos, noCheckOuterJoin); // reduce sink row resolver used to generate map join op RowResolver outputRS = opParseCtxMap.get(op).getRowResolver(); @@ -441,7 +435,7 @@ */ public static MapJoinOperator convertSMBJoinToMapJoin(HiveConf hconf, Map, OpParseContext> opParseCtxMap, - SMBMapJoinOperator smbJoinOp, QBJoinTree joinTree, int bigTablePos, boolean noCheckOuterJoin) + SMBMapJoinOperator smbJoinOp, int bigTablePos, boolean noCheckOuterJoin) throws SemanticException { // Create a new map join operator SMBJoinDesc smbJoinDesc = smbJoinOp.getConf(); @@ -488,7 +482,7 @@ } public MapJoinOperator generateMapJoinOperator(ParseContext pctx, JoinOperator op, - QBJoinTree joinTree, int mapJoinPos) throws SemanticException { + int mapJoinPos) throws SemanticException { HiveConf hiveConf = pctx.getConf(); boolean noCheckOuterJoin = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN) @@ -497,7 +491,8 @@ LinkedHashMap, OpParseContext> opParseCtxMap = pctx .getOpParseCtx(); MapJoinOperator mapJoinOp = convertMapJoin(pctx.getConf(), opParseCtxMap, op, - joinTree, mapJoinPos, noCheckOuterJoin, true); + op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(), op.getConf().getMapAliases(), + mapJoinPos, noCheckOuterJoin, true); // create a dummy select to select all columns genSelectPlan(pctx, mapJoinOp); return mapJoinOp; @@ -626,7 +621,7 @@ SelectDesc select = new SelectDesc(exprs, outputs, false); - SelectOperator sel = (SelectOperator) putOpInsertMap(OperatorFactory.getAndMakeChild(select, + SelectOperator sel = (SelectOperator) putOpInsertMap(pctx, OperatorFactory.getAndMakeChild(select, new RowSchema(inputRR.getColumnInfos()), input), inputRR); sel.setColumnExprMap(colExprMap); @@ -643,24 +638,22 @@ * * @param op * join operator - * @param joinTree - * qb join tree * @return -1 if it cannot be converted to a map-side join, position of the map join node * otherwise */ - private int mapSideJoin(JoinOperator op, QBJoinTree joinTree) throws SemanticException { + private int mapSideJoin(JoinOperator op) throws SemanticException { int mapJoinPos = -1; - if (joinTree.isMapSideJoin()) { + if (op.getConf().isMapSideJoin()) { int pos = 0; // In a map-side join, exactly one table is not present in memory. // The client provides the list of tables which can be cached in memory // via a hint. - if (joinTree.getJoinSrc() != null) { + if (op.getConf().isLeftInputJoin()) { mapJoinPos = pos; } - for (String src : joinTree.getBaseSrc()) { + for (String src : op.getConf().getBaseSrc()) { if (src != null) { - if (!joinTree.getMapAliases().contains(src)) { + if (!op.getConf().getMapAliases().contains(src)) { if (mapJoinPos >= 0) { return -1; } @@ -675,7 +668,7 @@ // leaving some table from the list of tables to be cached if (mapJoinPos == -1) { throw new SemanticException(ErrorMsg.INVALID_MAPJOIN_HINT.getMsg( - Arrays.toString(joinTree.getBaseSrc()))); + Arrays.toString(op.getConf().getBaseSrc()))); } } @@ -691,36 +684,34 @@ */ @Override public ParseContext transform(ParseContext pactx) throws SemanticException { - pGraphContext = pactx; List listMapJoinOps = new ArrayList(); // traverse all the joins and convert them if necessary - if (pGraphContext.getJoinContext() != null) { - Map joinMap = new HashMap(); - Map mapJoinMap = pGraphContext.getMapJoinContext(); + if (pactx.getJoinOps() != null) { + Set joinMap = new HashSet(); + Set mapJoinMap = pactx.getMapJoinOps(); if (mapJoinMap == null) { - mapJoinMap = new HashMap(); - pGraphContext.setMapJoinContext(mapJoinMap); + mapJoinMap = new HashSet(); + pactx.setMapJoinOps(mapJoinMap); } - Set> joinCtx = pGraphContext.getJoinContext().entrySet(); - Iterator> joinCtxIter = joinCtx.iterator(); + Iterator joinCtxIter = pactx.getJoinOps().iterator(); while (joinCtxIter.hasNext()) { - Map.Entry joinEntry = joinCtxIter.next(); - JoinOperator joinOp = joinEntry.getKey(); - QBJoinTree qbJoin = joinEntry.getValue(); - int mapJoinPos = mapSideJoin(joinOp, qbJoin); + JoinOperator joinOp = joinCtxIter.next(); + int mapJoinPos = mapSideJoin(joinOp); if (mapJoinPos >= 0) { - MapJoinOperator mapJoinOp = generateMapJoinOperator(pactx, joinOp, qbJoin, mapJoinPos); + MapJoinOperator mapJoinOp = generateMapJoinOperator(pactx, joinOp, mapJoinPos); listMapJoinOps.add(mapJoinOp); - mapJoinMap.put(mapJoinOp, qbJoin); + mapJoinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); + mapJoinMap.add(mapJoinOp); } else { - joinMap.put(joinOp, qbJoin); + joinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); + joinMap.add(joinOp); } } // store the new joinContext - pGraphContext.setJoinContext(joinMap); + pactx.setJoinOps(joinMap); } // Go over the list and find if a reducer is not needed @@ -746,15 +737,15 @@ // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefault(), opRules, new MapJoinWalkerCtx( - listMapJoinOpsNoRed, pGraphContext)); + listMapJoinOpsNoRed, pactx)); GraphWalker ogw = new GenMapRedWalker(disp); ArrayList topNodes = new ArrayList(); topNodes.addAll(listMapJoinOps); ogw.startWalking(topNodes, null); - pGraphContext.setListMapJoinOpsNoReducer(listMapJoinOpsNoRed); - return pGraphContext; + pactx.setListMapJoinOpsNoReducer(listMapJoinOpsNoRed); + return pactx; } /** @@ -800,7 +791,7 @@ } Operator ch = parent.getChildOperators().get(0); if (ch instanceof MapJoinOperator) { - if (!nonSubqueryMapJoin(ctx.getpGraphContext(), (MapJoinOperator) ch, mapJoin)) { + if (!nonSubqueryMapJoin((MapJoinOperator) ch, mapJoin)) { if (ch.getParentOperators().indexOf(parent) == ((MapJoinOperator) ch).getConf() .getPosBigTable()) { // not come from the local branch @@ -820,11 +811,8 @@ } } - private boolean nonSubqueryMapJoin(ParseContext pGraphContext, MapJoinOperator mapJoin, - MapJoinOperator parentMapJoin) { - QBJoinTree joinTree = pGraphContext.getMapJoinContext().get(mapJoin); - QBJoinTree parentJoinTree = pGraphContext.getMapJoinContext().get(parentMapJoin); - if (joinTree.getJoinSrc() != null && joinTree.getJoinSrc().equals(parentJoinTree)) { + private boolean nonSubqueryMapJoin(MapJoinOperator mapJoin, MapJoinOperator parentMapJoin) { + if (mapJoin.getParentOperators().contains(parentMapJoin)) { return true; } return false; @@ -1028,15 +1016,15 @@ } - public static ObjectPair, Map>> getKeys(QBJoinTree joinTree, JoinOperator op) { + public static ObjectPair, Map>> getKeys( + boolean leftInputJoin, String[] baseSrc, JoinOperator op) { // Walk over all the sources (which are guaranteed to be reduce sink // operators). // The join outputs a concatenation of all the inputs. - QBJoinTree leftSrc = joinTree.getJoinSrc(); List oldReduceSinkParentOps = new ArrayList(op.getNumParent()); - if (leftSrc != null) { + if (leftInputJoin) { // assert mapJoinPos == 0; Operator parentOp = op.getParentOperators().get(0); assert parentOp.getParentOperators().size() == 1; @@ -1044,7 +1032,7 @@ } byte pos = 0; - for (String src : joinTree.getBaseSrc()) { + for (String src : baseSrc) { if (src != null) { Operator parentOp = op.getParentOperators().get(pos); assert parentOp.getParentOperators().size() == 1; @@ -1062,12 +1050,14 @@ keyExprMap.put(pos, keyCols); } - return new ObjectPair, Map>>(oldReduceSinkParentOps, keyExprMap); + return new ObjectPair, Map>>( + oldReduceSinkParentOps, keyExprMap); } public static MapJoinDesc getMapJoinDesc(HiveConf hconf, LinkedHashMap, OpParseContext> opParseCtxMap, - JoinOperator op, QBJoinTree joinTree, int mapJoinPos, boolean noCheckOuterJoin) throws SemanticException { + JoinOperator op, boolean leftInputJoin, String[] baseSrc, List mapAliases, + int mapJoinPos, boolean noCheckOuterJoin) throws SemanticException { JoinDesc desc = op.getConf(); JoinCondDesc[] condns = desc.getConds(); Byte[] tagOrder = desc.getTagOrder(); @@ -1084,7 +1074,8 @@ Map> valueExprs = op.getConf().getExprs(); Map> newValueExprs = new HashMap>(); - ObjectPair, Map>> pair = getKeys(joinTree, op); + ObjectPair, Map>> pair = + getKeys(leftInputJoin, baseSrc, op); List oldReduceSinkParentOps = pair.getFirst(); for (Map.Entry> entry : valueExprs.entrySet()) { byte tag = entry.getKey(); @@ -1174,8 +1165,8 @@ // create dumpfile prefix needed to create descriptor String dumpFilePrefix = ""; - if (joinTree.getMapAliases() != null) { - for (String mapAlias : joinTree.getMapAliases()) { + if (mapAliases != null) { + for (String mapAlias : mapAliases) { dumpFilePrefix = dumpFilePrefix + mapAlias; } dumpFilePrefix = dumpFilePrefix + "-" + PlanUtils.getCountForMapJoinDumpFilePrefix(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; @@ -42,7 +43,6 @@ import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -55,15 +55,12 @@ */ public class NonBlockingOpDeDupProc implements Transform { - private ParseContext pctx; - @Override public ParseContext transform(ParseContext pctx) throws SemanticException { - this.pctx = pctx; String SEL = SelectOperator.getOperatorName(); String FIL = FilterOperator.getOperatorName(); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", SEL + "%" + SEL + "%"), new SelectDedup()); + opRules.put(new RuleRegExp("R1", SEL + "%" + SEL + "%"), new SelectDedup(pctx)); opRules.put(new RuleRegExp("R2", FIL + "%" + FIL + "%"), new FilterDedup()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); @@ -76,6 +73,13 @@ } private class SelectDedup implements NodeProcessor { + + private ParseContext pctx; + + public SelectDedup (ParseContext pctx) { + this.pctx = pctx; + } + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @@ -178,28 +182,37 @@ } return true; } - } - /** - * Change existing references in the context to point from child to parent operator. - * @param cSEL child operator (to be removed, and merged into parent) - * @param pSEL parent operator - */ - private void fixContextReferences(SelectOperator cSEL, SelectOperator pSEL) { - Collection qbJoinTrees = new ArrayList(); - qbJoinTrees.addAll(pctx.getJoinContext().values()); - qbJoinTrees.addAll(pctx.getMapJoinContext().values()); - for (QBJoinTree qbJoinTree : qbJoinTrees) { - Map> aliasToOpInfo = qbJoinTree.getAliasToOpInfo(); - for (Map.Entry> entry : aliasToOpInfo.entrySet()) { - if (entry.getValue() == cSEL) { - aliasToOpInfo.put(entry.getKey(), pSEL); + /** + * Change existing references in the context to point from child to parent operator. + * @param cSEL child operator (to be removed, and merged into parent) + * @param pSEL parent operator + */ + private void fixContextReferences(SelectOperator cSEL, SelectOperator pSEL) { + Collection>> mapsAliasToOpInfo = + new ArrayList>>(); + for (JoinOperator joinOp : pctx.getJoinOps()) { + if (joinOp.getConf().getAliasToOpInfo() != null) { + mapsAliasToOpInfo.add(joinOp.getConf().getAliasToOpInfo()); } } + for (MapJoinOperator mapJoinOp : pctx.getMapJoinOps()) { + if (mapJoinOp.getConf().getAliasToOpInfo() != null) { + mapsAliasToOpInfo.add(mapJoinOp.getConf().getAliasToOpInfo()); + } + } + for (Map> aliasToOpInfo : mapsAliasToOpInfo) { + for (Map.Entry> entry : aliasToOpInfo.entrySet()) { + if (entry.getValue() == cSEL) { + aliasToOpInfo.put(entry.getKey(), pSEL); + } + } + } } } private class FilterDedup implements NodeProcessor { + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (working copy) @@ -143,8 +143,15 @@ } private boolean checkThreshold(FetchData data, int limit, ParseContext pctx) throws Exception { - if (limit > 0 && data.hasOnlyPruningFilter()) { - return true; + if (limit > 0) { + if (data.hasOnlyPruningFilter()) { + /* partitioned table + query has only pruning filters */ + return true; + } else if (data.isPartitioned() == false && data.isFiltered() == false) { + /* unpartitioned table + no filters */ + return true; + } + /* fall through */ } long threshold = HiveConf.getLongVar(pctx.getConf(), HiveConf.ConfVars.HIVEFETCHTASKCONVERSIONTHRESHOLD); @@ -176,7 +183,7 @@ if (!aggressive && qb.hasTableSample(alias)) { return null; } - Table table = pctx.getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); if (table == null) { return null; } @@ -228,6 +235,10 @@ if (op.getChildOperators() == null || op.getChildOperators().size() != 1) { return null; } + + if (op instanceof FilterOperator) { + fetch.setFiltered(true); + } } if (op instanceof FileSinkOperator) { @@ -279,6 +290,11 @@ || operator instanceof ScriptOperator) { return false; } + + if (operator instanceof FilterOperator) { + fetch.setFiltered(true); + } + if (!traversed.add(operator)) { return true; } @@ -314,6 +330,7 @@ // this is always non-null when conversion is completed private Operator fileSink; + private boolean filtered; private FetchData(TableScanOperator scanOp, ReadEntity parent, Table table, SplitSample splitSample) { this.scanOp = scanOp; @@ -337,10 +354,23 @@ /* * all filters were executed during partition pruning */ - public boolean hasOnlyPruningFilter() { + public final boolean hasOnlyPruningFilter() { return this.onlyPruningFilter; } + public final boolean isPartitioned() { + return this.table.isPartitioned(); + } + + /* there are filter operators in the pipeline */ + public final boolean isFiltered() { + return this.filtered; + } + + public final void setFiltered(boolean filtered) { + this.filtered = filtered; + } + private FetchWork convertToWork() throws HiveException { inputs.clear(); if (!table.isPartitioned()) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java (working copy) @@ -79,11 +79,13 @@ public class SkewJoinOptimizer implements Transform { private static final Log LOG = LogFactory.getLog(SkewJoinOptimizer.class.getName()); - private static ParseContext parseContext; public static class SkewJoinProc implements NodeProcessor { - public SkewJoinProc() { + private ParseContext parseContext; + + public SkewJoinProc(ParseContext parseContext) { super(); + this.parseContext = parseContext; } @Override @@ -165,23 +167,14 @@ return null; } - // have to create a QBJoinTree for the cloned join operator - QBJoinTree originJoinTree = parseContext.getJoinContext().get(joinOp); - QBJoinTree newJoinTree; - try { - newJoinTree = originJoinTree.clone(); - } catch (CloneNotSupportedException e) { - LOG.debug("QBJoinTree could not be cloned: ", e); - return null; - } - JoinOperator joinOpClone; if (processSelect) { joinOpClone = (JoinOperator)(currOpClone.getParentOperators().get(0)); } else { joinOpClone = (JoinOperator)currOpClone; } - parseContext.getJoinContext().put(joinOpClone, newJoinTree); + joinOpClone.getConf().cloneQBJoinTreeProps(joinOp.getConf()); + parseContext.getJoinOps().add(joinOpClone); List tableScanCloneOpsForJoin = new ArrayList(); @@ -202,9 +195,7 @@ for (Entry> topOp : topOps.entrySet()) { TableScanOperator tso = (TableScanOperator) topOp.getValue(); - Table origTable = parseContext.getTopToTable().get(ctx.getCloneTSOpMap().get(tso)); String tabAlias = tso.getConf().getAlias(); - parseContext.getTopToTable().put(tso, origTable); int initCnt = 1; String newAlias = "subquery" + initCnt + ":" + tabAlias; while (origTopOps.containsKey(newAlias)) { @@ -213,7 +204,7 @@ } parseContext.getTopOps().put(newAlias, tso); - setUpAlias(originJoinTree, newJoinTree, tabAlias, newAlias, tso); + setUpAlias(joinOp, joinOpClone, tabAlias, newAlias, tso); } // Now do a union of the select operators: selectOp and selectOpClone @@ -410,7 +401,7 @@ if (op instanceof TableScanOperator) { TableScanOperator tsOp = (TableScanOperator)op; if (tableScanOpsForJoin.contains(tsOp)) { - return parseContext.getTopToTable().get(tsOp); + return tsOp.getConf().getTableMetadata(); } } if ((op.getParentOperators() == null) || (op.getParentOperators().isEmpty()) || @@ -629,19 +620,19 @@ /** * Set alias in the cloned join tree */ - private static void setUpAlias(QBJoinTree origin, QBJoinTree cloned, String origAlias, + private static void setUpAlias(JoinOperator origin, JoinOperator cloned, String origAlias, String newAlias, Operator topOp) { - cloned.getAliasToOpInfo().remove(origAlias); - cloned.getAliasToOpInfo().put(newAlias, topOp); - if (origin.getLeftAlias().equals(origAlias)) { - cloned.setLeftAlias(null); - cloned.setLeftAlias(newAlias); + cloned.getConf().getAliasToOpInfo().remove(origAlias); + cloned.getConf().getAliasToOpInfo().put(newAlias, topOp); + if (origin.getConf().getLeftAlias().equals(origAlias)) { + cloned.getConf().setLeftAlias(null); + cloned.getConf().setLeftAlias(newAlias); } - replaceAlias(origin.getLeftAliases(), cloned.getLeftAliases(), origAlias, newAlias); - replaceAlias(origin.getRightAliases(), cloned.getRightAliases(), origAlias, newAlias); - replaceAlias(origin.getBaseSrc(), cloned.getBaseSrc(), origAlias, newAlias); - replaceAlias(origin.getMapAliases(), cloned.getMapAliases(), origAlias, newAlias); - replaceAlias(origin.getStreamAliases(), cloned.getStreamAliases(), origAlias, newAlias); + replaceAlias(origin.getConf().getLeftAliases(), cloned.getConf().getLeftAliases(), origAlias, newAlias); + replaceAlias(origin.getConf().getRightAliases(), cloned.getConf().getRightAliases(), origAlias, newAlias); + replaceAlias(origin.getConf().getBaseSrc(), cloned.getConf().getBaseSrc(), origAlias, newAlias); + replaceAlias(origin.getConf().getMapAliases(), cloned.getConf().getMapAliases(), origAlias, newAlias); + replaceAlias(origin.getConf().getStreamAliases(), cloned.getConf().getStreamAliases(), origAlias, newAlias); } private static void replaceAlias(String[] origin, String[] cloned, @@ -677,7 +668,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", "TS%.*RS%JOIN%"), getSkewJoinProc()); + opRules.put(new RuleRegExp("R1", "TS%.*RS%JOIN%"), getSkewJoinProc(pctx)); SkewJoinOptProcCtx skewJoinOptProcCtx = new SkewJoinOptProcCtx(pctx); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along @@ -692,8 +683,8 @@ return pctx; } - private NodeProcessor getSkewJoinProc() { - return new SkewJoinProc(); + private NodeProcessor getSkewJoinProc(ParseContext parseContext) { + return new SkewJoinProc(parseContext); } /** Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (working copy) @@ -143,7 +143,7 @@ return null; } - Table destTable = parseCtx.getFsopToTable().get(fsOp); + Table destTable = fsOp.getConf().getTable(); if (destTable == null) { LOG.debug("Bailing out of sort dynamic partition optimization as destination table is null"); return null; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapjoinProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapjoinProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapjoinProc.java (working copy) @@ -60,7 +60,7 @@ } if (convert) { - convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext, pGraphContext); + convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext); } return null; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java (working copy) @@ -44,10 +44,10 @@ SortBucketJoinProcCtx smbJoinContext = (SortBucketJoinProcCtx) procCtx; boolean convert = canConvertJoinToSMBJoin( - joinOp, smbJoinContext, pGraphContext); + joinOp, smbJoinContext); if (convert) { - convertJoinToSMBJoin(joinOp, smbJoinContext, pGraphContext); + convertJoinToSMBJoin(joinOp, smbJoinContext); } return null; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java (working copy) @@ -21,18 +21,18 @@ import java.util.LinkedHashMap; import java.util.List; -import com.google.common.base.Preconditions; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import com.google.common.base.Preconditions; + public class SparkMapJoinProcessor extends MapJoinProcessor { /** @@ -50,8 +50,8 @@ @Override public MapJoinOperator convertMapJoin(HiveConf conf, LinkedHashMap, OpParseContext> opParseCtxMap, - JoinOperator op, QBJoinTree joinTree, int bigTablePos, - boolean noCheckOuterJoin, + JoinOperator op, boolean leftSrc, String[] baseSrc, List mapAliases, + int bigTablePos, boolean noCheckOuterJoin, boolean validateMapJoinTree) throws SemanticException { // outer join cannot be performed on a table which is being cached @@ -65,7 +65,8 @@ // create the map-join operator MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf, opParseCtxMap, - op, joinTree, bigTablePos, noCheckOuterJoin); + op, op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(), + op.getConf().getMapAliases(), bigTablePos, noCheckOuterJoin); // 1. remove RS as parent for the big table branch // 2. remove old join op from child set of all the RSs Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (working copy) @@ -228,7 +228,7 @@ return null; } - Table tbl = pctx.getTopToTable().get(tsOp); + Table tbl = tsOp.getConf().getTableMetadata(); List oneRow = new ArrayList(); List ois = new ArrayList(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java (working copy) @@ -60,7 +60,7 @@ currentPos++; continue; } - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); long currentSize = 0; if (!table.isPartitioned()) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java (working copy) @@ -107,7 +107,7 @@ // that has both intermediate tables and query input tables as input tables, // we should be able to guess if this JoinOperator will be converted to a MapJoin // based on hive.auto.convert.join.noconditionaltask.size. - for (JoinOperator joinOp: pCtx.getJoinContext().keySet()) { + for (JoinOperator joinOp: pCtx.getJoinOps()) { boolean isAbleToGuess = true; boolean mayConvert = false; // Get total size and individual alias's size @@ -124,7 +124,7 @@ Set aliases = new LinkedHashSet(); for (TableScanOperator tsop : topOps) { - Table table = pCtx.getTopToTable().get(tsop); + Table table = tsop.getConf().getTableMetadata(); if (table == null) { // table should not be null. throw new SemanticException("The table of " + Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java (working copy) @@ -161,11 +161,10 @@ * if the optimization can be applied. If yes, we add the name of the top table to * the tsOpToProcess to apply rewrite later on. * */ - Map topToTable = parseContext.getTopToTable(); for (Map.Entry> entry : parseContext.getTopOps().entrySet()) { String alias = entry.getKey(); TableScanOperator topOp = (TableScanOperator) entry.getValue(); - Table table = topToTable.get(topOp); + Table table = topOp.getConf().getTableMetadata(); List indexes = tableToIndex.get(table); if (indexes.isEmpty()) { continue; @@ -232,12 +231,16 @@ supportedIndexes.add(AggregateIndexHandler.class.getName()); // query the metastore to know what columns we have indexed - Collection topTables = parseContext.getTopToTable().values(); + Collection> topTables = parseContext.getTopOps().values(); Map> indexes = new HashMap>(); - for (Table tbl : topTables){ - List tblIndexes = IndexUtils.getIndexes(tbl, supportedIndexes); - if (tblIndexes.size() > 0) { - indexes.put(tbl, tblIndexes); + for (Operator op : topTables) { + if (op instanceof TableScanOperator) { + TableScanOperator tsOP = (TableScanOperator) op; + List tblIndexes = IndexUtils.getIndexes(tsOP.getConf().getTableMetadata(), + supportedIndexes); + if (tblIndexes.size() > 0) { + indexes.put(tsOP.getConf().getTableMetadata(), tblIndexes); + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java (working copy) @@ -170,7 +170,6 @@ // Need to remove the original TableScanOperators from these data structures // and add new ones - Map topToTable = rewriteQueryCtx.getParseContext().getTopToTable(); Map> topOps = rewriteQueryCtx.getParseContext() .getTopOps(); Map, OpParseContext> opParseContext = rewriteQueryCtx @@ -181,13 +180,8 @@ // remove original TableScanOperator topOps.remove(alias); - topToTable.remove(scanOperator); opParseContext.remove(scanOperator); - // construct a new descriptor for the index table scan - TableScanDesc indexTableScanDesc = new TableScanDesc(); - indexTableScanDesc.setGatherStats(false); - String indexTableName = rewriteQueryCtx.getIndexName(); Table indexTableHandle = null; try { @@ -198,6 +192,10 @@ throw new SemanticException(e.getMessage(), e); } + // construct a new descriptor for the index table scan + TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle); + indexTableScanDesc.setGatherStats(false); + String k = indexTableName + Path.SEPARATOR; indexTableScanDesc.setStatsAggPrefix(k); scanOperator.setConf(indexTableScanDesc); @@ -227,12 +225,10 @@ } // Scan operator now points to other table - topToTable.put(scanOperator, indexTableHandle); scanOperator.getConf().setAlias(newAlias); scanOperator.setAlias(indexTableName); topOps.put(newAlias, scanOperator); opParseContext.put(scanOperator, operatorContext); - rewriteQueryCtx.getParseContext().setTopToTable((HashMap) topToTable); rewriteQueryCtx.getParseContext().setTopOps( (HashMap>) topOps); rewriteQueryCtx.getParseContext().setOpParseCtx( Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java (working copy) @@ -139,7 +139,7 @@ // Table scan operator. TableScanOperator top = (TableScanOperator)nd; - org.apache.hadoop.hive.ql.metadata.Table t = pctx.getTopToTable().get(top); + org.apache.hadoop.hive.ql.metadata.Table t = top.getConf().getTableMetadata(); Table tab = t.getTTable(); // Generate the mappings Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java (working copy) @@ -52,7 +52,7 @@ protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { LBOpPartitionWalkerCtx owc = (LBOpPartitionWalkerCtx) procCtx; - Table tbl = owc.getParseContext().getTopToTable().get(top); + Table tbl = top.getConf().getTableMetadata(); if (tbl.isPartitioned()) { // Run partition pruner to get partitions ParseContext parseCtx = owc.getParseContext(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (working copy) @@ -161,7 +161,7 @@ Object... nodeOutputs) throws SemanticException { TableScanOperator ts = (TableScanOperator)nd; AnnotateOpTraitsProcCtx opTraitsCtx = (AnnotateOpTraitsProcCtx)procCtx; - Table table = opTraitsCtx.getParseContext().getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); PrunedPartitionList prunedPartList = null; try { prunedPartList = Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java (working copy) @@ -34,7 +34,6 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker.TaskGraphWalkerContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MapWork; /** @@ -53,8 +52,7 @@ throws SemanticException; protected void replaceTaskWithConditionalTask( - Task currTask, ConditionalTask cndTsk, - PhysicalContext physicalContext) { + Task currTask, ConditionalTask cndTsk) { // add this task into task tree // set all parent tasks List> parentTasks = currTask.getParentTasks(); @@ -88,8 +86,7 @@ // Replace the task with the new task. Copy the children and parents of the old // task to the new task. protected void replaceTask( - Task currTask, Task newTask, - PhysicalContext physicalContext) { + Task currTask, Task newTask) { // add this task into task tree // set all parent tasks List> parentTasks = currTask.getParentTasks(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java (working copy) @@ -103,7 +103,7 @@ Map opRules = new LinkedHashMap(); opRules.put(new RuleRegExp("R1", SelectOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getSelProc()); - // Matches only GroupByOpeartors which are reducers, rather than map group by operators, + // Matches only GroupByOperators which are reducers, rather than map group by operators, // or multi group by optimization specific operators opRules.put(new RuleExactMatch("R2", GroupByOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getGroupByProc()); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java (working copy) @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; @@ -669,7 +670,7 @@ processGroupByReduceSink((ReduceSinkOperator) rop, gop, bctx); - return processGroupBy((ReduceSinkOperator)rop , gop, bctx); + return processGroupBy(rop , gop, bctx); } /** @@ -683,12 +684,16 @@ protected void processGroupByReduceSink(ReduceSinkOperator rop, GroupByOperator gop, BucketingSortingCtx bctx){ + GroupByDesc groupByDesc = gop.getConf(); String sortOrder = rop.getConf().getOrder(); List bucketCols = new ArrayList(); List sortCols = new ArrayList(); assert rop.getConf().getKeyCols().size() <= rop.getSchema().getSignature().size(); // Group by operators select the key cols, so no need to find them in the values for (int i = 0; i < rop.getConf().getKeyCols().size(); i++) { + if (groupByDesc.pruneGroupingSetId() && groupByDesc.getGroupingSetPosition() == i) { + continue; + } String colName = rop.getSchema().getSignature().get(i).getInternalName(); bucketCols.add(new BucketCol(colName, i)); sortCols.add(new SortCol(colName, i, sortOrder.charAt(i))); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (working copy) @@ -48,7 +48,6 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx; @@ -400,7 +399,6 @@ // get parseCtx for this Join Operator ParseContext parseCtx = physicalContext.getParseContext(); - QBJoinTree joinTree = parseCtx.getJoinContext().get(joinOp); // start to generate multiple map join tasks JoinDesc joinDesc = joinOp.getConf(); @@ -458,7 +456,9 @@ } currWork.setOpParseCtxMap(parseCtx.getOpParseCtx()); - currWork.setJoinTree(joinTree); + currWork.setLeftInputJoin(joinOp.getConf().isLeftInputJoin()); + currWork.setBaseSrc(joinOp.getConf().getBaseSrc()); + currWork.setMapAliases(joinOp.getConf().getMapAliases()); if (bigTablePosition >= 0) { // create map join task and set big table as bigTablePosition @@ -466,7 +466,7 @@ newTask.setTaskTag(Task.MAPJOIN_ONLY_NOBACKUP); newTask.setFetchSource(currTask.isFetchSource()); - replaceTask(currTask, newTask, physicalContext); + replaceTask(currTask, newTask); // Can this task be merged with the child task. This can happen if a big table is being // joined with multiple small tables on different keys @@ -522,7 +522,9 @@ listTasks.add(currTask); // clear JoinTree and OP Parse Context currWork.setOpParseCtxMap(null); - currWork.setJoinTree(null); + currWork.setLeftInputJoin(false); + currWork.setBaseSrc(null); + currWork.setMapAliases(null); // create conditional task and insert conditional task into task tree ConditionalWork cndWork = new ConditionalWork(listWorks); @@ -541,7 +543,7 @@ cndTsk.setResolverCtx(resolverCtx); // replace the current task with the new generated conditional task - replaceTaskWithConditionalTask(currTask, cndTsk, physicalContext); + replaceTaskWithConditionalTask(currTask, cndTsk); return cndTsk; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java (working copy) @@ -21,6 +21,8 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -58,9 +60,10 @@ */ public class NullScanTaskDispatcher implements Dispatcher { + static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName()); + private final PhysicalContext physicalContext; - private final Map rules; - static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName()); + private final Map rules; public NullScanTaskDispatcher(PhysicalContext context, Map rules) { super(); @@ -91,18 +94,6 @@ return desc; } - private List getPathsForAlias(MapWork work, String alias) { - List paths = new ArrayList(); - - for (Map.Entry> entry : work.getPathToAliases().entrySet()) { - if (entry.getValue().contains(alias)) { - paths.add(entry.getKey()); - } - } - - return paths; - } - private void processAlias(MapWork work, String path, ArrayList aliasesAffected, ArrayList aliases) { // the aliases that are allowed to map to a null scan. @@ -164,7 +155,15 @@ ParseContext parseContext = physicalContext.getParseContext(); WalkerCtx walkerCtx = new WalkerCtx(); - for (MapWork mapWork: task.getMapWork()) { + List mapWorks = new ArrayList(task.getMapWork()); + Collections.sort(mapWorks, new Comparator() { + @Override + public int compare(MapWork o1, MapWork o2) { + return o1.getName().compareTo(o2.getName()); + } + }); + + for (MapWork mapWork : mapWorks) { LOG.debug("Looking at: "+mapWork.getName()); Collection> topOperators = mapWork.getAliasToWork().values(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java (working copy) @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor; import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx; @@ -168,8 +167,7 @@ // create map join task and set big table as bigTablePosition private MapRedTask convertSMBTaskToMapJoinTask(MapredWork origWork, int bigTablePosition, - SMBMapJoinOperator smbJoinOp, - QBJoinTree joinTree) + SMBMapJoinOperator smbJoinOp) throws UnsupportedEncodingException, SemanticException { // deep copy a new mapred work MapredWork newWork = Utilities.clonePlan(origWork); @@ -178,7 +176,7 @@ .getParseContext().getConf()); // generate the map join operator; already checked the map join MapJoinOperator newMapJoinOp = - getMapJoinOperator(newTask, newWork, smbJoinOp, joinTree, bigTablePosition); + getMapJoinOperator(newTask, newWork, smbJoinOp, bigTablePosition); // The reducer needs to be restored - Consider a query like: // select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key; @@ -246,7 +244,6 @@ // get parseCtx for this Join Operator ParseContext parseCtx = physicalContext.getParseContext(); - QBJoinTree joinTree = parseCtx.getSmbMapJoinContext().get(originalSMBJoinOp); // Convert the work containing to sort-merge join into a work, as if it had a regular join. // Note that the operator tree is not changed - is still contains the SMB join, but the @@ -257,9 +254,13 @@ SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(currJoinWork); currWork.getMapWork().setOpParseCtxMap(parseCtx.getOpParseCtx()); - currWork.getMapWork().setJoinTree(joinTree); + currWork.getMapWork().setLeftInputJoin(originalSMBJoinOp.getConf().isLeftInputJoin()); + currWork.getMapWork().setBaseSrc(originalSMBJoinOp.getConf().getBaseSrc()); + currWork.getMapWork().setMapAliases(originalSMBJoinOp.getConf().getMapAliases()); currJoinWork.getMapWork().setOpParseCtxMap(parseCtx.getOpParseCtx()); - currJoinWork.getMapWork().setJoinTree(joinTree); + currJoinWork.getMapWork().setLeftInputJoin(originalSMBJoinOp.getConf().isLeftInputJoin()); + currJoinWork.getMapWork().setBaseSrc(originalSMBJoinOp.getConf().getBaseSrc()); + currJoinWork.getMapWork().setMapAliases(originalSMBJoinOp.getConf().getMapAliases()); // create conditional work list and task list List listWorks = new ArrayList(); @@ -296,7 +297,7 @@ // create map join task for the given big table position MapRedTask newTask = convertSMBTaskToMapJoinTask( - currJoinWork, bigTablePosition, newSMBJoinOp, joinTree); + currJoinWork, bigTablePosition, newSMBJoinOp); MapWork mapWork = newTask.getWork().getMapWork(); Operator parentOp = originalSMBJoinOp.getParentOperators().get(bigTablePosition); @@ -334,7 +335,9 @@ listTasks.add(currTask); // clear JoinTree and OP Parse Context currWork.getMapWork().setOpParseCtxMap(null); - currWork.getMapWork().setJoinTree(null); + currWork.getMapWork().setLeftInputJoin(false); + currWork.getMapWork().setBaseSrc(null); + currWork.getMapWork().setMapAliases(null); // create conditional task and insert conditional task into task tree ConditionalWork cndWork = new ConditionalWork(listWorks); @@ -353,7 +356,7 @@ cndTsk.setResolverCtx(resolverCtx); // replace the current task with the new generated conditional task - replaceTaskWithConditionalTask(currTask, cndTsk, physicalContext); + replaceTaskWithConditionalTask(currTask, cndTsk); return cndTsk; } @@ -426,7 +429,6 @@ private MapJoinOperator getMapJoinOperator(MapRedTask task, MapredWork work, SMBMapJoinOperator oldSMBJoinOp, - QBJoinTree joinTree, int mapJoinPos) throws SemanticException { SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(task.getWork()); @@ -437,7 +439,6 @@ // generate the map join operator return MapJoinProcessor.convertSMBJoinToMapJoin(physicalContext.getConf(), - opParseContextMap, newSMBJoinOp, - joinTree, mapJoinPos, true); + opParseContextMap, newSMBJoinOp, mapJoinPos, true); } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java (working copy) @@ -74,12 +74,12 @@ // Check whether the specified BaseWork's operator tree contains a operator // of the specified operator class private boolean containsOp(BaseWork work, Class clazz) { - Set> matchingOps = getOp(work, clazz); + Set> matchingOps = getOp(work, clazz); return matchingOps != null && !matchingOps.isEmpty(); } - public static Set> getOp(BaseWork work, Class clazz) { - Set> ops = new HashSet>(); + public static Set> getOp(BaseWork work, Class clazz) { + Set> ops = new HashSet>(); if (work instanceof MapWork) { Collection> opSet = ((MapWork) work).getAliasToWork().values(); Stack> opStack = new Stack>(); @@ -184,7 +184,7 @@ Context ctx = physicalContext.getContext(); for (BaseWork work : allBaseWorks) { - Set> ops = getOp(work, MapJoinOperator.class); + Set> ops = getOp(work, MapJoinOperator.class); if (ops == null || ops.isEmpty()) { continue; } @@ -213,7 +213,7 @@ } for (BaseWork parentWork : originalWork.getParents(work)) { - Set> hashTableSinkOps = + Set> hashTableSinkOps = getOp(parentWork, SparkHashTableSinkOperator.class); if (hashTableSinkOps == null || hashTableSinkOps.isEmpty()) { continue; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (working copy) @@ -247,6 +247,7 @@ supportedGenericUDFs.add(GenericUDFWhen.class); supportedGenericUDFs.add(GenericUDFCoalesce.class); supportedGenericUDFs.add(GenericUDFElt.class); + supportedGenericUDFs.add(GenericUDFInitCap.class); // For type casts supportedGenericUDFs.add(UDFToLong.class); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java (working copy) @@ -28,6 +28,7 @@ import java.util.Stack; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler; @@ -47,6 +48,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.MapredWork; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** * @@ -115,12 +117,14 @@ supportedIndexes.add(BitmapIndexHandler.class.getName()); // query the metastore to know what columns we have indexed - Collection
topTables = pctx.getTopToTable().values(); Map> indexes = new HashMap>(); - for (Map.Entry entry : pctx.getTopToTable().entrySet()) { - List tblIndexes = IndexUtils.getIndexes(entry.getValue(), supportedIndexes); - if (tblIndexes.size() > 0) { - indexes.put(entry.getKey(), tblIndexes); + for (Operator op : pctx.getTopOps().values()) { + if (op instanceof TableScanOperator) { + List tblIndexes = IndexUtils.getIndexes(((TableScanOperator) op).getConf() + .getTableMetadata(), supportedIndexes); + if (tblIndexes.size() > 0) { + indexes.put((TableScanOperator) op, tblIndexes); + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (working copy) @@ -138,7 +138,7 @@ */ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx, String alias) throws SemanticException { - return prune(parseCtx.getTopToTable().get(ts), parseCtx.getOpToPartPruner().get(ts), + return prune(ts.getConf().getTableMetadata(), parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias, parseCtx.getPrunedPartitions()); } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (working copy) @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.ql.optimizer.BucketMapjoinProc; import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; @@ -130,7 +129,6 @@ private int convertJoinBucketMapJoin(JoinOperator joinOp, MapJoinOperator mapJoinOp, OptimizeSparkProcContext context, int bigTablePosition) throws SemanticException { ParseContext parseContext = context.getParseContext(); - QBJoinTree joinTree = parseContext.getJoinContext().get(joinOp); List joinAliases = new ArrayList(); String baseBigAlias = null; Map> posToAliasMap = joinOp.getPosToAliasMap(); @@ -146,7 +144,10 @@ } mapJoinOp.setPosToAliasMap(posToAliasMap); BucketMapjoinProc.checkAndConvertBucketMapJoin( - parseContext, mapJoinOp, joinTree, baseBigAlias, joinAliases); + parseContext, + mapJoinOp, + baseBigAlias, + joinAliases); MapJoinDesc joinDesc = mapJoinOp.getConf(); return joinDesc.isBucketMapJoin() ? joinDesc.getBigTableBucketNumMapping().size() : -1; @@ -374,7 +375,8 @@ ParseContext parseContext = context.getParseContext(); MapJoinOperator mapJoinOp = MapJoinProcessor.convertJoinOpMapJoinOp(context.getConf(), parseContext.getOpParseCtx(), joinOp, - parseContext.getJoinContext().get(joinOp), bigTablePosition, true); + joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp.getConf().getMapAliases(), + bigTablePosition, true); Operator parentBigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition); @@ -393,6 +395,9 @@ } } + // Data structures + mapJoinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); + return mapJoinOp; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java (working copy) @@ -70,7 +70,7 @@ if (convert) { removeSmallTableReduceSink(mapJoinOp); - convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext, pGraphContext); + convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext); } return null; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinProcFactory.java (working copy) @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.optimizer.physical.SkewJoinProcFactory; import org.apache.hadoop.hive.ql.optimizer.physical.SparkMapJoinResolver; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils; import org.apache.hadoop.hive.ql.plan.BaseWork; @@ -138,16 +137,22 @@ String streamDesc = taskTmpDir.toUri().toString(); if (GenMapRedUtils.needsTagging((ReduceWork) childWork)) { Operator childReducer = ((ReduceWork) childWork).getReducer(); - QBJoinTree joinTree = null; + String id = null; if (childReducer instanceof JoinOperator) { - joinTree = parseContext.getJoinContext().get(childReducer); + if (parseContext.getJoinOps().contains(childReducer)) { + id = ((JoinOperator)childReducer).getConf().getId(); + } } else if (childReducer instanceof MapJoinOperator) { - joinTree = parseContext.getMapJoinContext().get(childReducer); + if (parseContext.getMapJoinOps().contains(childReducer)) { + id = ((MapJoinOperator)childReducer).getConf().getId(); + } } else if (childReducer instanceof SMBMapJoinOperator) { - joinTree = parseContext.getSmbMapJoinContext().get(childReducer); + if (parseContext.getSmbMapJoinOps().contains(childReducer)) { + id = ((SMBMapJoinOperator)childReducer).getConf().getId(); + } } - if (joinTree != null && joinTree.getId() != null) { - streamDesc = joinTree.getId() + ":$INTNAME"; + if (id != null) { + streamDesc = id + ":$INTNAME"; } else { streamDesc = "$INTNAME"; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java (working copy) @@ -65,7 +65,7 @@ joinOp, smbJoinContext, pGraphContext, stack); if (convert) { - return convertJoinToSMBJoinAndReturn(joinOp, smbJoinContext, pGraphContext); + return convertJoinToSMBJoinAndReturn(joinOp, smbJoinContext); } return null; } @@ -76,7 +76,7 @@ if (!supportBucketMapJoin(stack)) { return false; } - return canConvertJoinToSMBJoin(joinOperator, smbJoinContext, pGraphContext); + return canConvertJoinToSMBJoin(joinOperator, smbJoinContext); } //Preliminary checks. In the MR version of the code, these used to be done via another walk, @@ -102,11 +102,10 @@ protected SMBMapJoinOperator convertJoinToSMBJoinAndReturn( JoinOperator joinOp, - SortBucketJoinProcCtx smbJoinContext, - ParseContext parseContext) throws SemanticException { - MapJoinOperator mapJoinOp = convertJoinToBucketMapJoin(joinOp, smbJoinContext, parseContext); + SortBucketJoinProcCtx smbJoinContext) throws SemanticException { + MapJoinOperator mapJoinOp = convertJoinToBucketMapJoin(joinOp, smbJoinContext); SMBMapJoinOperator smbMapJoinOp = - convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext, parseContext); + convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext); smbMapJoinOp.setConvertedAutomaticallySMBJoin(true); return smbMapJoinOp; } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (working copy) @@ -103,7 +103,7 @@ AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; PrunedPartitionList partList = aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop); - Table table = aspCtx.getParseContext().getTopToTable().get(tsop); + Table table = tsop.getConf().getTableMetadata(); try { // gather statistics for the first time and the attach it to table scan operator Index: ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (working copy) @@ -1734,6 +1734,23 @@ // 1. Gather GB Expressions (AST) (GB + Aggregations) // NOTE: Multi Insert is not supported String detsClauseName = qbp.getClauseNames().iterator().next(); + // Check and transform group by *. This will only happen for select distinct *. + // Here the "genSelectPlan" is being leveraged. + // The main benefits are (1) remove virtual columns that should + // not be included in the group by; (2) add the fully qualified column names to unParseTranslator + // so that view is supported. The drawback is that an additional SEL op is added. If it is + // not necessary, it will be removed by NonBlockingOpDeDupProc Optimizer because it will match + // SEL%SEL% rule. + ASTNode selExprList = qb.getParseInfo().getSelForClause(detsClauseName); + if (selExprList.getToken().getType() == HiveParser.TOK_SELECTDI + && selExprList.getChildCount() == 1 && selExprList.getChild(0).getChildCount() == 1) { + ASTNode node = (ASTNode) selExprList.getChild(0).getChild(0); + if (node.getToken().getType() == HiveParser.TOK_ALLCOLREF) { + srcRel = genSelectLogicalPlan(qb, srcRel, srcRel); + RowResolver rr = this.relToHiveRR.get(srcRel); + qbp.setSelExprForClause(detsClauseName, SemanticAnalyzer.genSelectDIAST(rr)); + } + } List grpByAstExprs = SemanticAnalyzer.getGroupByForClause(qbp, detsClauseName); HashMap aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); boolean hasGrpByAstExprs = (grpByAstExprs != null && !grpByAstExprs.isEmpty()) ? true : false; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java (working copy) @@ -17,16 +17,20 @@ */ package org.apache.hadoop.hive.ql.parse; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; public class ColumnAccessAnalyzer { - private static final Log LOG = LogFactory.getLog(ColumnAccessAnalyzer.class.getName()); + private static final Log LOG = LogFactory.getLog(ColumnAccessAnalyzer.class.getName()); private final ParseContext pGraphContext; public ColumnAccessAnalyzer() { @@ -39,19 +43,22 @@ public ColumnAccessInfo analyzeColumnAccess() throws SemanticException { ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo(); - Map topOps = pGraphContext.getTopToTable(); - for (TableScanOperator op : topOps.keySet()) { - Table table = topOps.get(op); - String tableName = table.getCompleteName(); - List referenced = op.getReferencedColumns(); - for (String column : referenced) { - columnAccessInfo.add(tableName, column); - } - if (table.isPartitioned()) { - PrunedPartitionList parts = pGraphContext.getPrunedPartitions(table.getTableName(), op); - if (parts.getReferredPartCols() != null) { - for (String partKey : parts.getReferredPartCols()) { - columnAccessInfo.add(tableName, partKey); + Collection> topOps = pGraphContext.getTopOps().values(); + for (Operator op : topOps) { + if (op instanceof TableScanOperator) { + TableScanOperator top = (TableScanOperator) op; + Table table = top.getConf().getTableMetadata(); + String tableName = table.getCompleteName(); + List referenced = top.getReferencedColumns(); + for (String column : referenced) { + columnAccessInfo.add(tableName, column); + } + if (table.isPartitioned()) { + PrunedPartitionList parts = pGraphContext.getPrunedPartitions(table.getTableName(), top); + if (parts.getReferredPartCols() != null) { + for (String partKey : parts.getReferredPartCols()) { + columnAccessInfo.add(tableName, partKey); + } } } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (working copy) @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Task; @@ -68,7 +69,7 @@ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, "Target is not a directory : " + toURI)); } else { - FileStatus[] files = fs.listStatus(toPath); + FileStatus[] files = fs.listStatus(toPath, FileUtils.HIDDEN_FILES_PATH_FILTER); if (files != null && files.length != 0) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, "Target is not an empty directory : " + toURI)); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (working copy) @@ -191,10 +191,7 @@ setupMapWork(mapWork, context, partitions, root, alias); - if (context.parseContext != null - && context.parseContext.getTopToTable() != null - && context.parseContext.getTopToTable().containsKey(ts) - && context.parseContext.getTopToTable().get(ts).isDummyTable()) { + if (ts.getConf().getTableMetadata() != null && ts.getConf().getTableMetadata().isDummyTable()) { mapWork.setDummyTableScan(true); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (working copy) @@ -2216,7 +2216,7 @@ @after { popMsg(state); } : KW_INSERT KW_OVERWRITE destination ifNotExists? -> ^(TOK_DESTINATION destination ifNotExists?) - | KW_INSERT KW_INTO KW_TABLE tableOrPartition + | KW_INSERT KW_INTO KW_TABLE? tableOrPartition -> ^(TOK_INSERT_INTO tableOrPartition) ; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; @@ -357,7 +358,7 @@ throws IOException, SemanticException { LOG.debug("checking emptiness of " + targetPath.toString()); if (fs.exists(targetPath)) { - FileStatus[] status = fs.listStatus(targetPath); + FileStatus[] status = fs.listStatus(targetPath, FileUtils.HIDDEN_FILES_PATH_FILTER); if (status.length > 0) { LOG.debug("Files inc. " + status[0].getPath().toString() + " found in path : " + targetPath.toString()); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (working copy) @@ -926,18 +926,17 @@ */ for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) { ColumnInfo cInfo = new ColumnInfo(inpCInfo); - String colAlias = cInfo.getAlias(); - String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName()); - if (tabColAlias != null) { - colAlias = tabColAlias[1]; - } - ASTNode inExpr = null; - inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR); + ASTNode inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR); if (inExpr != null) { rr.putExpression(inExpr, cInfo); } else { - rr.put(cInfo.getTabAlias(), colAlias, cInfo); + String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName()); + if (tabColAlias != null) { + rr.put(tabColAlias[0], tabColAlias[1], cInfo); + } else { + rr.put(inpCInfo.getTabAlias(), inpCInfo.getAlias(), cInfo); + } } String[] altMapping = inputRR.getAlternateMappings(inpCInfo.getInternalName()); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (working copy) @@ -31,7 +31,6 @@ import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.ListSinkOperator; @@ -43,8 +42,6 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.hooks.LineageInfo; import org.apache.hadoop.hive.ql.hooks.ReadEntity; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -73,15 +70,11 @@ private HashMap opToSamplePruner; private Map> opToPartToSkewedPruner; private HashMap> topOps; - private HashMap> topSelOps; private LinkedHashMap, OpParseContext> opParseCtx; - private Map joinContext; - private Map mapJoinContext; - private Map smbMapJoinContext; - private HashMap topToTable; - private Map fsopToTable; + private Set joinOps; + private Set mapJoinOps; + private Set smbMapJoinOps; private List reduceSinkOperatorsAddedByEnforceBucketingSorting; - private HashMap> topToProps; private HashMap nameToSplitSample; private List loadTableWork; private List loadFileWork; @@ -128,15 +121,11 @@ * @param opToPartList * @param topOps * list of operators for the top query - * @param topSelOps - * list of operators for the selects introduced for column pruning * @param opParseCtx * operator parse context - contains a mapping from operator to * operator parse state (row resolver etc.) - * @param joinContext + * @param joinOps * context needed join processing (map join specifically) - * @param topToTable - * the top tables being processed * @param loadTableWork * list of destination tables being loaded * @param loadFileWork @@ -163,13 +152,9 @@ HashMap opToPartPruner, HashMap opToPartList, HashMap> topOps, - HashMap> topSelOps, LinkedHashMap, OpParseContext> opParseCtx, - Map joinContext, - Map smbMapJoinContext, - HashMap topToTable, - HashMap> topToProps, - Map fsopToTable, + Set joinOps, + Set smbMapJoinOps, List loadTableWork, List loadFileWork, Context ctx, HashMap idToTableNameMap, int destTableId, UnionProcContext uCtx, List> listMapJoinOpsNoReducer, @@ -188,16 +173,12 @@ this.ast = ast; this.opToPartPruner = opToPartPruner; this.opToPartList = opToPartList; - this.joinContext = joinContext; - this.smbMapJoinContext = smbMapJoinContext; - this.topToTable = topToTable; - this.fsopToTable = fsopToTable; - this.topToProps = topToProps; + this.joinOps = joinOps; + this.smbMapJoinOps = smbMapJoinOps; this.loadFileWork = loadFileWork; this.loadTableWork = loadTableWork; this.opParseCtx = opParseCtx; this.topOps = topOps; - this.topSelOps = topSelOps; this.ctx = ctx; this.idToTableNameMap = idToTableNameMap; this.destTableId = destTableId; @@ -297,29 +278,6 @@ return opToPartList; } - /** - * @return the topToTable - */ - public HashMap getTopToTable() { - return topToTable; - } - - /** - * @param topToTable - * the topToTable to set - */ - public void setTopToTable(HashMap topToTable) { - this.topToTable = topToTable; - } - - public Map getFsopToTable() { - return fsopToTable; - } - - public void setFsopToTable(Map fsopToTable) { - this.fsopToTable = fsopToTable; - } - public List getReduceSinkOperatorsAddedByEnforceBucketingSorting() { return reduceSinkOperatorsAddedByEnforceBucketingSorting; } @@ -331,21 +289,6 @@ } /** - * @return the topToProps - */ - public HashMap> getTopToProps() { - return topToProps; - } - - /** - * @param topToProps - * the topToProps to set - */ - public void setTopToProps(HashMap> topToProps) { - this.topToProps = topToProps; - } - - /** * @return the topOps */ public HashMap> getTopOps() { @@ -361,22 +304,6 @@ } /** - * @return the topSelOps - */ - public HashMap> getTopSelOps() { - return topSelOps; - } - - /** - * @param topSelOps - * the topSelOps to set - */ - public void setTopSelOps( - HashMap> topSelOps) { - this.topSelOps = topSelOps; - } - - /** * @return the opParseCtx */ public LinkedHashMap, OpParseContext> getOpParseCtx() { @@ -476,18 +403,18 @@ } /** - * @return the joinContext + * @return the joinOps */ - public Map getJoinContext() { - return joinContext; + public Set getJoinOps() { + return joinOps; } /** - * @param joinContext - * the joinContext to set + * @param joinOps + * the joinOps to set */ - public void setJoinContext(Map joinContext) { - this.joinContext = joinContext; + public void setJoinOps(Set joinOps) { + this.joinOps = joinOps; } /** @@ -570,20 +497,20 @@ return lInfo; } - public Map getMapJoinContext() { - return mapJoinContext; + public Set getMapJoinOps() { + return mapJoinOps; } - public void setMapJoinContext(Map mapJoinContext) { - this.mapJoinContext = mapJoinContext; + public void setMapJoinOps(Set mapJoinOps) { + this.mapJoinOps = mapJoinOps; } - public Map getSmbMapJoinContext() { - return smbMapJoinContext; + public Set getSmbMapJoinOps() { + return smbMapJoinOps; } - public void setSmbMapJoinContext(Map smbMapJoinContext) { - this.smbMapJoinContext = smbMapJoinContext; + public void setSmbMapJoinOps(Set smbMapJoinOps) { + this.smbMapJoinOps = smbMapJoinOps; } public GlobalLimitCtx getGlobalLimitCtx() { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java (working copy) @@ -75,7 +75,7 @@ TableScanOperator tableScan = (TableScanOperator) nd; ParseContext parseContext = context.parseContext; - Class inputFormat = parseContext.getTopToTable().get(tableScan) + Class inputFormat = tableScan.getConf().getTableMetadata() .getInputFormatClass(); QB queryBlock = parseContext.getQB(); QBParseInfo parseInfo = parseContext.getQB().getParseInfo(); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.Serializable; +import java.security.AccessControlException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,14 +39,17 @@ import java.util.regex.PatternSyntaxException; import org.antlr.runtime.ClassicToken; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeWizard; import org.antlr.runtime.tree.TreeWizard.ContextVisitor; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -200,9 +204,12 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.security.UserGroupInformation; /** * Implementation of the semantic analyzer. It generates the query plan. @@ -374,12 +381,9 @@ opToPartList = pctx.getOpToPartList(); opToSamplePruner = pctx.getOpToSamplePruner(); topOps = pctx.getTopOps(); - topSelOps = pctx.getTopSelOps(); opParseCtx = pctx.getOpParseCtx(); loadTableWork = pctx.getLoadTableWork(); loadFileWork = pctx.getLoadFileWork(); - joinContext = pctx.getJoinContext(); - smbMapJoinContext = pctx.getSmbMapJoinContext(); ctx = pctx.getContext(); destTableId = pctx.getDestTableId(); idToTableNameMap = pctx.getIdToTableNameMap(); @@ -393,15 +397,15 @@ } public ParseContext getParseContext() { - return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList, topOps, - topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable, topToTableProps, - fsopToTable, loadTableWork, - loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, + return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList, + topOps, opParseCtx, + new HashSet(joinContext.keySet()), + new HashSet(smbMapJoinContext.keySet()), + loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, - queryProperties); + reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); } @SuppressWarnings("nls") @@ -1669,7 +1673,7 @@ throw new SemanticException(e); } try { - fname = ctx.getExternalTmpPath( + fname = ctx.getExtTmpPathRelTo( FileUtils.makeQualified(location, conf)).toString(); } catch (Exception e) { throw new SemanticException(generateErrorMessage(ast, @@ -1685,8 +1689,9 @@ } else { // This is the only place where isQuery is set to true; it defaults to false. qb.setIsQuery(true); - fname = ctx.getMRTmpPath().toString(); - ctx.setResDir(new Path(fname)); + Path stagingPath = getStagingDirectoryPathname(qb); + fname = stagingPath.toString(); + ctx.setResDir(stagingPath); } } qb.getMetaData().setDestForAlias(name, fname, @@ -1742,6 +1747,160 @@ } } + /** + * Checks if a given path is encrypted (valid only for HDFS files) + * @param path The path to check for encryption + * @return True if the path is encrypted; False if it is not encrypted + * @throws HiveException If an error occurs while checking for encryption + */ + private boolean isPathEncrypted(Path path) throws HiveException { + HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; + + hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); + if (hdfsEncryptionShim != null) { + try { + if (hdfsEncryptionShim.isPathEncrypted(path)) { + return true; + } + } catch (Exception e) { + throw new HiveException("Unable to determine if " + path + "is encrypted: " + e, e); + } + } + + return false; + } + + /** + * Compares to path key encryption strenghts. + * + * @param p1 Path to an HDFS file system + * @param p2 Path to an HDFS file system + * @return -1 if strength is weak; 0 if is equals; 1 if it is stronger + * @throws HiveException If an error occurs while comparing key strengths. + */ + private int comparePathKeyStrength(Path p1, Path p2) throws HiveException { + HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; + + hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); + if (hdfsEncryptionShim != null) { + try { + return hdfsEncryptionShim.comparePathKeyStrength(p1, p2); + } catch (Exception e) { + throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); + } + } + + return 0; // Non-encrypted path (or equals strength) + } + + /** + * Checks if a given path has read-only access permissions. + * + * @param path The path to check for read-only permissions. + * @return True if the path is read-only; False otherwise. + * @throws HiveException If an error occurs while checking file permissions. + */ + private boolean isPathReadOnly(Path path) throws HiveException { + HiveConf conf = SessionState.get().getConf(); + try { + FileSystem fs = path.getFileSystem(conf); + UserGroupInformation ugi = Utils.getUGI(); + FileStatus status = fs.getFileStatus(path); + + // We just check for writing permissions. If it fails with AccessControException, then it + // means the location may be read-only. + FileUtils.checkFileAccessWithImpersonation(fs, status, FsAction.WRITE, ugi.getUserName()); + + // Path has writing permissions + return false; + } catch (AccessControlException e) { + // An AccessControlException may be caused for other different errors, + // but we take it as if our path is read-only + return true; + } catch (Exception e) { + throw new HiveException("Unable to determine if " + path + " is read only: " + e, e); + } + } + + /** + * Gets the strongest encrypted table path. + * + * @param qb The QB object that contains a list of all table locations. + * @return The strongest encrypted path + * @throws HiveException if an error occurred attempting to compare the encryption strength + */ + private Path getStrongestEncryptedTablePath(QB qb) throws HiveException { + List tabAliases = new ArrayList(qb.getTabAliases()); + Path strongestPath = null; + + /* Walk through all found table locations to get the most encrypted table */ + for (String alias : tabAliases) { + Table tab = qb.getMetaData().getTableForAlias(alias); + if (tab != null) { + Path tablePath = tab.getDataLocation(); + if (tablePath != null) { + try { + if (strongestPath == null) { + strongestPath = tablePath; + } else if ("hdfs".equals(tablePath.toUri().getScheme()) + && isPathEncrypted(tablePath) + && comparePathKeyStrength(tablePath, strongestPath) > 0) + { + strongestPath = tablePath; + } + } catch (HiveException e) { + throw new HiveException("Unable to find the most secure table path: " + e, e); + } + } + } + } + + return strongestPath; + } + + /** + * Gets the staging directory where MR files will be stored temporary. + * It walks through the QB plan to find the correct location where save temporary files. This + * temporary location (or staging directory) may be created inside encrypted tables locations for + * security reasons. If the QB has read-only tables, then the older scratch directory will be used, + * or a permission error will be thrown if the requested query table is encrypted and the old scratch + * directory is not. + * + * @param qb The QB object that contains a list of all table locations. + * @return The path to the staging directory. + * @throws HiveException If an error occurs while identifying the correct staging location. + */ + private Path getStagingDirectoryPathname(QB qb) throws HiveException { + Path stagingPath = null, tablePath; + + // Looks for the most encrypted table location (if there is one) + tablePath = getStrongestEncryptedTablePath(qb); + if (tablePath != null && isPathEncrypted(tablePath)) { + // Only HDFS paths can be checked for encryption + if ("hdfs".equals(tablePath.toUri().getScheme())) { + if (isPathReadOnly(tablePath)) { + Path tmpPath = ctx.getMRTmpPath(); + if (comparePathKeyStrength(tablePath, tmpPath) < 0) { + throw new HiveException("Read-only encrypted tables cannot be read " + + "if the scratch directory is not encrypted (or encryption is weak)"); + } else { + stagingPath = tmpPath; + } + } + } else { + LOG.debug("Encryption is not applicable to table path " + tablePath.toString()); + } + + if (stagingPath == null) { + stagingPath = ctx.getMRTmpPath(tablePath.toUri()); + } + } else { + stagingPath = ctx.getMRTmpPath(); + } + + return stagingPath; + } + private void replaceViewReferenceWithDefinition(QB qb, Table tab, String tab_name, String alias) throws SemanticException { @@ -2642,7 +2801,7 @@ @SuppressWarnings("nls") // TODO: make aliases unique, otherwise needless rewriting takes place - Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, + Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, ArrayList col_list, HashSet excludeCols, RowResolver input, RowResolver colSrcRR, Integer pos, RowResolver output, List aliases, boolean ensureUniqueCols) throws SemanticException { @@ -3938,7 +4097,7 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false, groupByMemoryUsage, memoryThreshold, null, false, 0, numDistinctUDFs > 0), + false, groupByMemoryUsage, memoryThreshold, null, false, -1, numDistinctUDFs > 0), new RowSchema(groupByOutputRowResolver.getColumnInfos()), input), groupByOutputRowResolver); op.setColumnExprMap(colExprMap); @@ -4063,10 +4222,11 @@ } // This is only needed if a new grouping set key is being created - int groupingSetsPosition = 0; + int groupingSetsPosition = -1; // For grouping sets, add a dummy grouping key if (groupingSetsPresent) { + groupingSetsPosition = groupByKeys.size(); // Consider the query: select a,b, count(1) from T group by a,b with cube; // where it is being executed in a single map-reduce job // The plan is TableScan -> GroupBy1 -> ReduceSink -> GroupBy2 -> FileSink @@ -4081,7 +4241,6 @@ colExprMap); } else { - groupingSetsPosition = groupByKeys.size(); // The grouping set has not yet been processed. Create a new grouping key // Consider the query: select a,b, count(1) from T group by a,b with cube; // where it is being executed in 2 map-reduce jobs @@ -4297,7 +4456,7 @@ } // The grouping set key is present after the grouping keys, before the distinct keys - int groupingSetsPosition = groupByKeys.size(); + int groupingSetsPosition = -1; // For grouping sets, add a dummy grouping key // This dummy key needs to be added as a reduce key @@ -4309,6 +4468,7 @@ // This function is called for GroupBy1 to create an additional grouping key // for the grouping set (corresponding to the rollup). if (groupingSetsPresent) { + groupingSetsPosition = groupByKeys.size(); createNewGroupingKey(groupByKeys, outputColumnNames, groupByOutputRowResolver, @@ -4865,8 +5025,10 @@ colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1)); } + int groupingSetsPosition = -1; // For grouping sets, add a dummy grouping key if (groupingSetsPresent) { + groupingSetsPosition = groupByKeys.size(); addGroupingSetKey( groupByKeys, groupByInputRowResolver2, @@ -4922,7 +5084,8 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false, groupByMemoryUsage, memoryThreshold, null, false, 0, containsDistinctAggr), + false, groupByMemoryUsage, memoryThreshold, null, false, + groupingSetsPosition, containsDistinctAggr), new RowSchema(groupByOutputRowResolver2.getColumnInfos()), reduceSinkOperatorInfo2), groupByOutputRowResolver2); op.setColumnExprMap(colExprMap); @@ -5857,6 +6020,7 @@ Table dest_tab = null; // destination table if any boolean destTableIsAcid = false; // should the destination table be written to using ACID + boolean destTableIsTemporary = false; Partition dest_part = null;// destination partition if any Path queryTmpdir = null; // the intermediate destination directory Path dest_path = null; // the final destination directory @@ -5874,6 +6038,7 @@ dest_tab = qbm.getDestTableForAlias(dest); destTableIsAcid = isAcidTable(dest_tab); + destTableIsTemporary = dest_tab.isTemporary(); // Is the user trying to insert into a external tables if ((!conf.getBoolVar(HiveConf.ConfVars.HIVE_INSERT_INTO_EXTERNAL_TABLES)) && @@ -5946,7 +6111,7 @@ if (isNonNativeTable) { queryTmpdir = dest_path; } else { - queryTmpdir = ctx.getExternalTmpPath(dest_path); + queryTmpdir = ctx.getExtTmpPathRelTo(dest_path); } if (dpCtx != null) { // set the root of the temporary path where dynamic partition columns will populate @@ -6127,7 +6292,7 @@ try { Path qPath = FileUtils.makeQualified(dest_path, conf); - queryTmpdir = ctx.getExternalTmpPath(qPath); + queryTmpdir = ctx.getExtTmpPathRelTo(qPath); } catch (Exception e) { throw new SemanticException("Error creating temporary folder on: " + dest_path, e); @@ -6143,6 +6308,7 @@ CreateTableDesc tblDesc = qb.getTableDesc(); if (tblDesc != null) { field_schemas = new ArrayList(); + destTableIsTemporary = tblDesc.isTemporary(); } boolean first = true; @@ -6287,6 +6453,8 @@ fileSinkDesc.setWriteType(wt); acidFileSinks.add(fileSinkDesc); } + + fileSinkDesc.setTemporary(destTableIsTemporary); /* Set List Bucketing context. */ if (lbCtx != null) { @@ -6305,7 +6473,7 @@ // it should be the same as the MoveWork's sourceDir. fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString()); if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { - String statsTmpLoc = ctx.getExternalTmpPath(queryTmpdir).toString(); + String statsTmpLoc = ctx.getExtTmpPathRelTo(queryTmpdir).toString(); LOG.info("Set stats collection dir : " + statsTmpLoc); conf.set(StatsSetupConst.STATS_TMP_LOC, statsTmpLoc); } @@ -6334,7 +6502,9 @@ + dest_path + " row schema: " + inputRR.toString()); } - fsopToTable.put((FileSinkOperator) output, dest_tab); + FileSinkOperator fso = (FileSinkOperator) output; + fso.getConf().setTable(dest_tab); + fsopToTable.put(fso, dest_tab); return output; } @@ -6867,12 +7037,11 @@ } - @SuppressWarnings("nls") private Operator genReduceSinkPlan(String dest, QB qb, Operator input, int numReducers) throws SemanticException { - + RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - + // First generate the expression for the partition and sort keys // The cluster by clause / distribute by clause has the aliases for // partition function @@ -6880,15 +7049,14 @@ if (partitionExprs == null) { partitionExprs = qb.getParseInfo().getDistributeByForClause(dest); } - ArrayList partitionCols = new ArrayList(); + ArrayList partCols = new ArrayList(); if (partitionExprs != null) { int ccount = partitionExprs.getChildCount(); for (int i = 0; i < ccount; ++i) { ASTNode cl = (ASTNode) partitionExprs.getChild(i); - partitionCols.add(genExprNodeDesc(cl, inputRR)); + partCols.add(genExprNodeDesc(cl, inputRR)); } } - ASTNode sortExprs = qb.getParseInfo().getClusterByForClause(dest); if (sortExprs == null) { sortExprs = qb.getParseInfo().getSortByForClause(dest); @@ -6908,11 +7076,7 @@ } } } - Operator dummy = Operator.createDummy(); - dummy.setParentOperators(Arrays.asList(input)); - ArrayList sortCols = new ArrayList(); - ArrayList sortColsBack = new ArrayList(); StringBuilder order = new StringBuilder(); if (sortExprs != null) { int ccount = sortExprs.getChildCount(); @@ -6933,9 +7097,25 @@ } ExprNodeDesc exprNode = genExprNodeDesc(cl, inputRR); sortCols.add(exprNode); - sortColsBack.add(ExprNodeDescUtils.backtrack(exprNode, dummy, input)); } } + return genReduceSinkPlan(input, partCols, sortCols, order.toString(), numReducers); + } + + @SuppressWarnings("nls") + private Operator genReduceSinkPlan(Operator input, + ArrayList partitionCols, ArrayList sortCols, + String sortOrder, int numReducers) throws SemanticException { + + RowResolver inputRR = opParseCtx.get(input).getRowResolver(); + + Operator dummy = Operator.createDummy(); + dummy.setParentOperators(Arrays.asList(input)); + + ArrayList sortColsBack = new ArrayList(); + for (ExprNodeDesc sortCol : sortCols) { + sortColsBack.add(ExprNodeDescUtils.backtrack(sortCol, dummy, input)); + } // For the generation of the values expression just get the inputs // signature and generate field expressions for those RowResolver rsRR = new RowResolver(); @@ -6993,7 +7173,7 @@ // TODO Not 100% sure NOT_ACID is always right here. ReduceSinkDesc rsdesc = PlanUtils.getReduceSinkDesc(sortCols, valueCols, outputColumns, - false, -1, partitionCols, order.toString(), numReducers, AcidUtils.Operation.NOT_ACID); + false, -1, partitionCols, sortOrder, numReducers, AcidUtils.Operation.NOT_ACID); Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(rsdesc, new RowSchema(rsRR.getColumnInfos()), input), rsRR); @@ -7341,6 +7521,7 @@ JoinOperator joinOp = (JoinOperator) genJoinOperatorChildren(joinTree, joinSrcOp, srcOps, omitOpts, joinKeys); + joinOp.getConf().setQBJoinTreeProps(joinTree); joinContext.put(joinOp, joinTree); Operator op = joinOp; @@ -7434,7 +7615,7 @@ .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD); Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false, groupByMemoryUsage, memoryThreshold, null, false, 0, false), + false, groupByMemoryUsage, memoryThreshold, null, false, -1, false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), inputOperatorInfo), groupByOutputRowResolver); @@ -8751,6 +8932,23 @@ // insert a select operator here used by the ColumnPruner to reduce // the data to shuffle curr = insertSelectAllPlanForGroupBy(curr); + // Check and transform group by *. This will only happen for select distinct *. + // Here the "genSelectPlan" is being leveraged. + // The main benefits are (1) remove virtual columns that should + // not be included in the group by; (2) add the fully qualified column names to unParseTranslator + // so that view is supported. The drawback is that an additional SEL op is added. If it is + // not necessary, it will be removed by NonBlockingOpDeDupProc Optimizer because it will match + // SEL%SEL% rule. + ASTNode selExprList = qbp.getSelForClause(dest); + if (selExprList.getToken().getType() == HiveParser.TOK_SELECTDI + && selExprList.getChildCount() == 1 && selExprList.getChild(0).getChildCount() == 1) { + ASTNode node = (ASTNode) selExprList.getChild(0).getChild(0); + if (node.getToken().getType() == HiveParser.TOK_ALLCOLREF) { + curr = genSelectPlan(dest, qb, curr, curr); + RowResolver rr = opParseCtx.get(curr).getRowResolver(); + qbp.setSelExprForClause(dest, SemanticAnalyzer.genSelectDIAST(rr)); + } + } if (conf.getBoolVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE)) { if (!conf.getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) { curr = genGroupByPlanMapAggrNoSkew(dest, qb, curr); @@ -9249,7 +9447,7 @@ } // Create the root of the operator tree - TableScanDesc tsDesc = new TableScanDesc(alias, vcList); + TableScanDesc tsDesc = new TableScanDesc(alias, vcList, tab); setupStats(tsDesc, qb.getParseInfo(), tab, alias, rwsch); SplitSample sample = nameToSplitSample.get(alias_id); @@ -9271,6 +9469,7 @@ Map props = qb.getTabPropsForAlias(alias); if (props != null) { topToTableProps.put((TableScanOperator) top, props); + tsDesc.setOpProps(props); } } else { rwsch = opParseCtx.get(top).getRowResolver(); @@ -9433,7 +9632,7 @@ tsDesc.setGatherStats(false); } else { if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { - String statsTmpLoc = ctx.getExternalTmpPath(tab.getPath()).toString(); + String statsTmpLoc = ctx.getExtTmpPathRelTo(tab.getPath()).toString(); LOG.info("Set stats collection dir : " + statsTmpLoc); conf.set(StatsSetupConst.STATS_TMP_LOC, statsTmpLoc); } @@ -9964,9 +10163,11 @@ } // 4. Generate Parse Context for Optimizer & Physical compiler - ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child, opToPartPruner, opToPartList, - topOps, topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable, topToTableProps, - fsopToTable, loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, + ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child, + opToPartPruner, opToPartList, topOps, opParseCtx, + new HashSet(joinContext.keySet()), + new HashSet(smbMapJoinContext.keySet()), + loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); @@ -11948,130 +12149,34 @@ } private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, - RowResolver inputRR, - Operator input) throws SemanticException{ + RowResolver inputRR, Operator input) throws SemanticException{ + ArrayList partCols = new ArrayList(); - ArrayList valueCols = new ArrayList(); ArrayList orderCols = new ArrayList(); - Map colExprMap = new HashMap(); - List outputColumnNames = new ArrayList(); - StringBuilder orderString = new StringBuilder(); + StringBuilder order = new StringBuilder(); - ArrayList partColList = spec.getQueryPartitionSpec().getExpressions(); - for (PartitionExpression partCol : partColList) { + for (PartitionExpression partCol : spec.getQueryPartitionSpec().getExpressions()) { ExprNodeDesc partExpr = genExprNodeDesc(partCol.getExpression(), inputRR); partCols.add(partExpr); orderCols.add(partExpr); - orderString.append('+'); + order.append('+'); } - ArrayList orderColList = spec.getQueryOrderSpec() == null ? - new ArrayList() : - spec.getQueryOrderSpec().getExpressions(); - for (int i = 0; i < orderColList.size(); i++) { - OrderExpression orderCol = orderColList.get(i); - org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order order = orderCol.getOrder(); - if (order.name().equals("ASC")) { - orderString.append('+'); - } else { - orderString.append('-'); - } - ExprNodeDesc orderExpr = genExprNodeDesc(orderCol.getExpression(), inputRR); - orderCols.add(orderExpr); - } - - ArrayList colInfoList = inputRR.getColumnInfos(); - RowResolver rsNewRR = new RowResolver(); - int pos = 0; - for (ColumnInfo colInfo : colInfoList) { - ExprNodeDesc valueColExpr = new ExprNodeColumnDesc(colInfo); - valueCols.add(valueColExpr); - String internalName = SemanticAnalyzer.getColumnInternalName(pos++); - outputColumnNames.add(internalName); - colExprMap.put(internalName, valueColExpr); - - String[] alias = inputRR.reverseLookup(colInfo.getInternalName()); - ColumnInfo newColInfo = new ColumnInfo( - internalName, colInfo.getType(), alias[0], - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); - rsNewRR.put(alias[0], alias[1], newColInfo); - String[] altMapping = inputRR.getAlternateMappings(colInfo.getInternalName()); - if ( altMapping != null ) { - rsNewRR.put(altMapping[0], altMapping[1], newColInfo); + if (spec.getQueryOrderSpec() != null) { + for (OrderExpression orderCol : spec.getQueryOrderSpec().getExpressions()) { + String orderString = orderCol.getOrder().name(); + if (orderString.equals("ASC")) { + order.append('+'); + } else { + order.append('-'); } - } - - input = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils - .getReduceSinkDesc(orderCols, - valueCols, outputColumnNames, false, - -1, partCols, orderString.toString(), -1, AcidUtils.Operation.NOT_ACID), - new RowSchema(rsNewRR.getColumnInfos()), input), rsNewRR); - input.setColumnExprMap(colExprMap); - - - // Construct the RR for extract operator - RowResolver extractRR = new RowResolver(); - LinkedHashMap colsAddedByHaving = - new LinkedHashMap(); - pos = 0; - - for (ColumnInfo colInfo : colInfoList) { - String[] alias = inputRR.reverseLookup(colInfo.getInternalName()); - /* - * if we have already encountered this colInfo internalName. - * We encounter it again because it must be put for the Having clause. - * We will add these entries in the end; in a loop on colsAddedByHaving. See below. - */ - if ( colsAddedByHaving.containsKey(alias)) { - continue; + orderCols.add(genExprNodeDesc(orderCol.getExpression(), inputRR)); } - ASTNode astNode = PTFTranslator.getASTNode(colInfo, inputRR); - ColumnInfo eColInfo = new ColumnInfo( - SemanticAnalyzer.getColumnInternalName(pos++), colInfo.getType(), alias[0], - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); - - if ( astNode == null ) { - extractRR.put(alias[0], alias[1], eColInfo); - } - else { - /* - * in case having clause refers to this column may have been added twice; - * once with the ASTNode.toStringTree as the alias - * and then with the real alias. - */ - extractRR.putExpression(astNode, eColInfo); - if ( !astNode.toStringTree().toLowerCase().equals(alias[1]) ) { - colsAddedByHaving.put(alias, eColInfo); - } - } - String[] altMapping = inputRR.getAlternateMappings(colInfo.getInternalName()); - if ( altMapping != null ) { - extractRR.put(altMapping[0], altMapping[1], eColInfo); - } } - for(Map.Entry columnAddedByHaving : colsAddedByHaving.entrySet() ) { - String[] alias = columnAddedByHaving.getKey(); - ColumnInfo eColInfo = columnAddedByHaving.getValue(); - extractRR.put(alias[0], alias[1], eColInfo); - } - - /* - * b. Construct Extract Operator. - */ - input = putOpInsertMap(OperatorFactory.getAndMakeChild( - new ExtractDesc( - new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE - .toString(), "", false)), - new RowSchema(inputRR.getColumnInfos()), - input), extractRR); - - - return input; + return genReduceSinkPlan(input, partCols, orderCols, order.toString(), -1); } - public static ArrayList parseSelect(String selectExprStr) throws SemanticException { @@ -12207,4 +12312,25 @@ protected boolean deleting() { return false; } + public static ASTNode genSelectDIAST(RowResolver rr) { + HashMap> map = rr.getRslvMap(); + ASTNode selectDI = new ASTNode(new CommonToken(HiveParser.TOK_SELECTDI, "TOK_SELECTDI")); + for (String tabAlias : map.keySet()) { + for (Entry entry : map.get(tabAlias).entrySet()) { + selectDI.addChild(buildSelExprSubTree(tabAlias, entry.getKey())); + } + } + return selectDI; + } + private static ASTNode buildSelExprSubTree(String tableAlias, String col) { + ASTNode selexpr = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + ASTNode tableOrCol = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, + "TOK_TABLE_OR_COL")); + ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, ".")); + tableOrCol.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, tableAlias))); + dot.addChild(tableOrCol); + dot.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, col))); + selexpr.addChild(dot); + return selexpr; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java (working copy) @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.parse; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -143,7 +142,7 @@ // Must be deterministic order map for consistent q-test output across Java versions Map> tableToKeysMap = new LinkedHashMap>(); - Table tbl = pGraphContext.getTopToTable().get(tso); + Table tbl = tso.getConf().getTableMetadata(); tableToKeysMap.put(tbl.getCompleteName(), keyColNames); tableAccessCtx.addOperatorTableAccess(op, tableToKeysMap); @@ -174,10 +173,9 @@ // Get the key column names for each side of the join, // and check if the keys are all constants // or columns (not expressions). If yes, proceed. - QBJoinTree joinTree = pGraphContext.getJoinContext().get(op); - assert(parentOps.size() == joinTree.getBaseSrc().length); + assert(parentOps.size() == op.getConf().getBaseSrc().length); int pos = 0; - for (String src : joinTree.getBaseSrc()) { + for (String src : op.getConf().getBaseSrc()) { if (src != null) { assert(parentOps.get(pos) instanceof ReduceSinkOperator); ReduceSinkOperator reduceSinkOp = (ReduceSinkOperator) parentOps.get(pos); @@ -203,7 +201,7 @@ return null; } - Table tbl = pGraphContext.getTopToTable().get(tso); + Table tbl = tso.getConf().getTableMetadata(); tableToKeysMap.put(tbl.getCompleteName(), keyColNames); } else { return null; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (working copy) @@ -26,8 +26,6 @@ import java.util.List; import java.util.Set; -import com.google.common.collect.Interner; -import com.google.common.collect.Interners; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -60,6 +58,9 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; + /** * TaskCompiler is a the base class for classes that compile * operator pipelines into tasks. @@ -386,9 +387,7 @@ ParseContext clone = new ParseContext(conf, pCtx.getQB(), pCtx.getParseTree(), pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(), - pCtx.getTopSelOps(), pCtx.getOpParseCtx(), pCtx.getJoinContext(), - pCtx.getSmbMapJoinContext(), pCtx.getTopToTable(), pCtx.getTopToProps(), - pCtx.getFsopToTable(), + pCtx.getOpParseCtx(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(), pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(), pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(), pCtx.getListMapJoinOpsNoReducer(), pCtx.getGroupOpToInputTables(), @@ -399,7 +398,7 @@ pCtx.getQueryProperties()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); - clone.setMapJoinContext(pCtx.getMapJoinContext()); + clone.setMapJoinOps(pCtx.getMapJoinOps()); return clone; } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java (working copy) @@ -97,6 +97,10 @@ int tokenStartIndex = node.getTokenStartIndex(); int tokenStopIndex = node.getTokenStopIndex(); + if (tokenStopIndex < 0) { + // this is for artificially added tokens + return; + } Translation translation = new Translation(); translation.tokenStopIndex = tokenStopIndex; translation.replacementText = replacementText; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java (working copy) @@ -82,7 +82,7 @@ ParseContext parseContext = context.parseContext; @SuppressWarnings("rawtypes") - Class inputFormat = parseContext.getTopToTable().get(tableScan) + Class inputFormat = tableScan.getConf().getTableMetadata() .getInputFormatClass(); QB queryBlock = parseContext.getQB(); QBParseInfo parseInfo = parseContext.getQB().getParseInfo(); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java (working copy) @@ -18,12 +18,22 @@ package org.apache.hadoop.hive.ql.plan; + +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.PTFUtils; + public class AbstractOperatorDesc implements OperatorDesc { protected boolean vectorMode = false; protected transient Statistics statistics; protected transient OpTraits opTraits; + protected transient Map opProps; + static { + PTFUtils.makeTransient(AbstractOperatorDesc.class, "opProps"); + } + @Override @Explain(skipHeader = true, displayName = "Statistics") public Statistics getStatistics() { @@ -51,4 +61,12 @@ public void setOpTraits(OpTraits opTraits) { this.opTraits = opTraits; } + + public Map getOpProps() { + return opProps; + } + + public void setOpProps(Map props) { + this.opProps = props; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java (working copy) @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.Table; /** * FileSinkDesc. @@ -47,6 +48,7 @@ private String compressCodec; private String compressType; private boolean multiFileSpray; + private boolean temporary; // Whether the files output by this FileSink can be merged, e.g. if they are to be put into a // bucketed or sorted table/partition they cannot be merged. private boolean canBeMerged; @@ -89,6 +91,8 @@ private AcidUtils.Operation writeType = AcidUtils.Operation.NOT_ACID; private long txnId = 0; // transaction id for this operation + private transient Table table; + public FileSinkDesc() { } @@ -217,7 +221,22 @@ public void setMultiFileSpray(boolean multiFileSpray) { this.multiFileSpray = multiFileSpray; } + + /** + * @return destination is temporary + */ + public boolean isTemporary() { + return temporary; + } + /** + * @param totalFiles the totalFiles to set + */ + public void setTemporary(boolean temporary) { + this.temporary = temporary; + } + + public boolean canBeMerged() { return canBeMerged; } @@ -421,4 +440,12 @@ public long getTransactionId() { return txnId; } + + public Table getTable() { + return table; + } + + public void setTable(Table table) { + this.table = table; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java (working copy) @@ -61,7 +61,7 @@ private ArrayList keys; private List listGroupingSets; private boolean groupingSetsPresent; - private int groupingSetPosition; + private int groupingSetPosition = -1; private ArrayList aggregators; private ArrayList outputColumnNames; private float groupByMemoryUsage; @@ -177,6 +177,12 @@ return outputColumnNames; } + @Explain(displayName = "pruneGroupingSetId", displayOnlyOnTrue = true) + public boolean pruneGroupingSetId() { + return groupingSetPosition >= 0 && + outputColumnNames.size() != keys.size() + aggregators.size(); + } + public void setOutputColumnNames( ArrayList outputColumnNames) { this.outputColumnNames = outputColumnNames; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (working copy) @@ -27,6 +27,8 @@ import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.QBJoinTree; /** * Join operator Descriptor implementation. @@ -88,12 +90,26 @@ // used only for explain. private transient ExprNodeDesc [][] joinKeys; + + // Data structures coming originally from QBJoinTree + private transient String leftAlias; + private transient String[] leftAliases; + private transient String[] rightAliases; + private transient String[] baseSrc; + private transient String id; + private transient boolean mapSideJoin; + private transient List mapAliases; //map-side join aliases + private transient Map> aliasToOpInfo; + private transient boolean leftInputJoin; + private transient List streamAliases; + public JoinDesc() { } public JoinDesc(final Map> exprs, List outputColumnNames, final boolean noOuterJoin, - final JoinCondDesc[] conds, final Map> filters, ExprNodeDesc[][] joinKeys) { + final JoinCondDesc[] conds, final Map> filters, + ExprNodeDesc[][] joinKeys) { this.exprs = exprs; this.outputColumnNames = outputColumnNames; this.noOuterJoin = noOuterJoin; @@ -509,4 +525,88 @@ public void setFixedAsSorted(boolean fixedAsSorted) { this.fixedAsSorted = fixedAsSorted; } + + public String[] getLeftAliases() { + return leftAliases; + } + + public String[] getBaseSrc() { + return baseSrc; + } + + public String getId() { + return id; + } + + public List getMapAliases() { + return mapAliases; + } + + public Map> getAliasToOpInfo() { + return aliasToOpInfo; + } + + public boolean isLeftInputJoin() { + return leftInputJoin; + } + + public String getLeftAlias() { + return leftAlias; + } + + public void setLeftAlias(String leftAlias) { + this.leftAlias = leftAlias; + } + + public String[] getRightAliases() { + return rightAliases; + } + + public List getStreamAliases() { + return streamAliases; + } + + public boolean isMapSideJoin() { + return mapSideJoin; + } + + public void setQBJoinTreeProps(JoinDesc joinDesc) { + leftAlias = joinDesc.leftAlias; + leftAliases = joinDesc.leftAliases; + rightAliases = joinDesc.rightAliases; + baseSrc = joinDesc.baseSrc; + id = joinDesc.id; + mapSideJoin = joinDesc.mapSideJoin; + mapAliases = joinDesc.mapAliases; + aliasToOpInfo = joinDesc.aliasToOpInfo; + leftInputJoin = joinDesc.leftInputJoin; + streamAliases = joinDesc.streamAliases; + } + + public void setQBJoinTreeProps(QBJoinTree joinTree) { + leftAlias = joinTree.getLeftAlias(); + leftAliases = joinTree.getLeftAliases(); + rightAliases = joinTree.getRightAliases(); + baseSrc = joinTree.getBaseSrc(); + id = joinTree.getId(); + mapSideJoin = joinTree.isMapSideJoin(); + mapAliases = joinTree.getMapAliases(); + aliasToOpInfo = joinTree.getAliasToOpInfo(); + leftInputJoin = joinTree.getJoinSrc() != null; + streamAliases = joinTree.getStreamAliases(); + } + + public void cloneQBJoinTreeProps(JoinDesc joinDesc) { + leftAlias = joinDesc.leftAlias; + leftAliases = joinDesc.leftAliases == null ? null : joinDesc.leftAliases.clone(); + rightAliases = joinDesc.rightAliases == null ? null : joinDesc.rightAliases.clone(); + baseSrc = joinDesc.baseSrc == null ? null : joinDesc.baseSrc.clone(); + id = joinDesc.id; + mapSideJoin = joinDesc.mapSideJoin; + mapAliases = joinDesc.mapAliases == null ? null : new ArrayList(joinDesc.mapAliases); + aliasToOpInfo = new HashMap>(joinDesc.aliasToOpInfo); + leftInputJoin = joinDesc.leftInputJoin; + streamAliases = joinDesc.streamAliases == null ? null : new ArrayList(joinDesc.streamAliases); + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (working copy) @@ -30,7 +30,6 @@ import java.util.Map.Entry; import java.util.Set; -import com.google.common.collect.Interner; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -41,10 +40,11 @@ import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.BucketCol; import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.SortCol; import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SplitSample; import org.apache.hadoop.mapred.JobConf; +import com.google.common.collect.Interner; + /** * MapWork represents all the information used to run a map task on the cluster. * It is first used when the query planner breaks the logical plan into tasks and @@ -105,8 +105,10 @@ public static final int SAMPLING_ON_START = 2; // sampling on task running // the following two are used for join processing - private QBJoinTree joinTree; private LinkedHashMap, OpParseContext> opParseCtxMap; + private boolean leftInputJoin; + private String[] baseSrc; + private List mapAliases; private boolean mapperCannotSpanPartns; @@ -419,14 +421,6 @@ return useOneNullRowInputFormat; } - public QBJoinTree getJoinTree() { - return joinTree; - } - - public void setJoinTree(QBJoinTree joinTree) { - this.joinTree = joinTree; - } - public void setMapperCannotSpanPartns(boolean mapperCannotSpanPartns) { this.mapperCannotSpanPartns = mapperCannotSpanPartns; } @@ -579,4 +573,28 @@ public boolean getDoSplitsGrouping() { return this.doSplitsGrouping; } + + public boolean isLeftInputJoin() { + return leftInputJoin; + } + + public void setLeftInputJoin(boolean leftInputJoin) { + this.leftInputJoin = leftInputJoin; + } + + public String[] getBaseSrc() { + return baseSrc; + } + + public void setBaseSrc(String[] baseSrc) { + this.baseSrc = baseSrc; + } + + public List getMapAliases() { + return mapAliases; + } + + public void setMapAliases(List mapAliases) { + this.mapAliases = mapAliases; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.Map; public interface OperatorDesc extends Serializable, Cloneable { public Object clone() throws CloneNotSupportedException; @@ -26,4 +27,5 @@ public void setStatistics(Statistics statistics); public OpTraits getOpTraits(); public void setOpTraits(OpTraits opTraits); + public Map getOpProps(); } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -956,7 +956,7 @@ ReadEntity parentViewInfo = getParentViewInfo(alias, parseCtx.getViewAliasToInput()); // Adds tables only for create view (PPD filter can be appended by outer query) - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo)); } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (working copy) @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; /** @@ -36,7 +37,7 @@ private static final long serialVersionUID = 1L; static { - PTFUtils.makeTransient(TableScanDesc.class, "filterObject", "referencedColumns"); + PTFUtils.makeTransient(TableScanDesc.class, "filterObject", "referencedColumns", "tableMetadata"); } private String alias; @@ -93,23 +94,32 @@ private boolean isMetadataOnly = false; - @SuppressWarnings("nls") + private transient final Table tableMetadata; + + public TableScanDesc() { + this(null, null); } - public TableScanDesc(final String alias) { - this.alias = alias; + @SuppressWarnings("nls") + public TableScanDesc(Table tblMetadata) { + this(null, tblMetadata); } - public TableScanDesc(final String alias, List vcs) { + public TableScanDesc(final String alias, Table tblMetadata) { + this(alias, null, tblMetadata); + } + + public TableScanDesc(final String alias, List vcs, Table tblMetadata) { this.alias = alias; this.virtualCols = vcs; + this.tableMetadata = tblMetadata; } @Override public Object clone() { List vcs = new ArrayList(getVirtualCols()); - return new TableScanDesc(getAlias(), vcs); + return new TableScanDesc(getAlias(), vcs, this.tableMetadata); } @Explain(displayName = "alias") @@ -250,8 +260,12 @@ public void setIsMetadataOnly(boolean metadata_only) { isMetadataOnly = metadata_only; } - + public boolean getIsMetadataOnly() { return isMetadataOnly; } + + public Table getTableMetadata() { + return tableMetadata; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (working copy) @@ -897,7 +897,7 @@ HiveConf hiveConf) { TableScanDesc tableScanDesc = tableScanOp.getConf(); - Table tbl = owi.getParseContext().getTopToTable().get(tableScanOp); + Table tbl = tableScanDesc.getTableMetadata(); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER)) { // attach the original predicate to the table scan operator for index // optimizations that require the pushed predicate before pcr & later Index: ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState; /** @@ -49,8 +50,14 @@ } public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) - throws SQLException { - HiveCommand hiveCommand = HiveCommand.find(cmd); + throws SQLException { + return getForHiveCommandInternal(cmd, conf, false); + } + + public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf conf, + boolean testOnly) + throws SQLException { + HiveCommand hiveCommand = HiveCommand.find(cmd, testOnly); if (hiveCommand == null || isBlank(cmd[0])) { return null; } @@ -58,7 +65,8 @@ conf = new HiveConf(); } Set availableCommands = new HashSet(); - for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST).split(",")) { + for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST) + .split(",")) { availableCommands.add(availableCommand.toLowerCase().trim()); } if (!availableCommands.contains(cmd[0].trim().toLowerCase())) { @@ -82,6 +90,12 @@ return new CompileProcessor(); case RELOAD: return new ReloadProcessor(); + case CRYPTO: + try { + return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf); + } catch (HiveException e) { + throw new SQLException("Fail to start the command processor due to the exception: ", e); + } default: throw new AssertionError("Unknown HiveCommand " + hiveCommand); } Index: ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java (working copy) @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.processors; + +import org.apache.commons.cli.Options; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.HadoopShims; + +import java.io.IOException; +import java.util.Arrays; + +/** + * This class processes HADOOP commands used for HDFS encryption. It is meant to be run + * only by Hive unit & queries tests. + */ +public class CryptoProcessor implements CommandProcessor { + public static final Log LOG = LogFactory.getLog(CryptoProcessor.class.getName()); + + private HadoopShims.HdfsEncryptionShim encryptionShim; + + private Options CREATE_KEY_OPTIONS; + private Options DELETE_KEY_OPTIONS; + private Options CREATE_ZONE_OPTIONS; + + private int DEFAULT_BIT_LENGTH = 128; + + private HiveConf conf; + + public CryptoProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { + this.encryptionShim = encryptionShim; + this.conf = conf; + + CREATE_KEY_OPTIONS = new Options(); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("bitLength").create()); // optional + + DELETE_KEY_OPTIONS = new Options(); + DELETE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + + CREATE_ZONE_OPTIONS = new Options(); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("path").isRequired().create()); + } + + private CommandLine parseCommandArgs(final Options opts, String[] args) throws ParseException { + CommandLineParser parser = new GnuParser(); + return parser.parse(opts, args); + } + + private CommandProcessorResponse returnErrorResponse(final String errmsg) { + return new CommandProcessorResponse(1, "Encryption Processor Helper Failed:" + errmsg, null); + } + + private void writeTestOutput(final String msg) { + SessionState.get().out.println(msg); + } + + @Override + public void init() { + } + + @Override + public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + String[] args = command.split("\\s+"); + + if (args.length < 1) { + return returnErrorResponse("Command arguments are empty."); + } + + if (encryptionShim == null) { + return returnErrorResponse("Hadoop encryption shim is not initialized."); + } + + String action = args[0]; + String params[] = Arrays.copyOfRange(args, 1, args.length); + + try { + if (action.equalsIgnoreCase("create_key")) { + createEncryptionKey(params); + } else if (action.equalsIgnoreCase("create_zone")) { + createEncryptionZone(params); + } else if (action.equalsIgnoreCase("delete_key")) { + deleteEncryptionKey(params); + } else { + return returnErrorResponse("Unknown command action: " + action); + } + } catch (Exception e) { + return returnErrorResponse(e.getMessage()); + } + + return new CommandProcessorResponse(0); + } + + /** + * Creates an encryption key using the parameters passed through the 'create_key' action. + * + * @param params Parameters passed to the 'create_key' command action. + * @throws Exception If key creation failed. + */ + private void createEncryptionKey(String[] params) throws Exception { + CommandLine args = parseCommandArgs(CREATE_KEY_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); + + try { + encryptionShim.createKey(keyName, new Integer(bitLength)); + } catch (Exception e) { + throw new Exception("Cannot create encryption key: " + e.getMessage()); + } + + writeTestOutput("Encryption key created: '" + keyName + "'"); + } + + /** + * Creates an encryption zone using the parameters passed through the 'create_zone' action. + * + * @param params Parameters passed to the 'create_zone' command action. + * @throws Exception If zone creation failed. + */ + private void createEncryptionZone(String[] params) throws Exception { + CommandLine args = parseCommandArgs(CREATE_ZONE_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + Path cryptoZone = new Path(args.getOptionValue("path")); + if (cryptoZone == null) { + throw new Exception("Cannot create encryption zone: Invalid path '" + + args.getOptionValue("path") + "'"); + } + + try { + encryptionShim.createEncryptionZone(cryptoZone, keyName); + } catch (IOException e) { + throw new Exception("Cannot create encryption zone: " + e.getMessage()); + } + + writeTestOutput("Encryption zone created: '" + cryptoZone + "' using key: '" + keyName + "'"); + } + + /** + * Deletes an encryption key using the parameters passed through the 'delete_key' action. + * + * @param params Parameters passed to the 'delete_key' command action. + * @throws Exception If key deletion failed. + */ + private void deleteEncryptionKey(String[] params) throws Exception { + CommandLine args = parseCommandArgs(DELETE_KEY_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + try { + encryptionShim.deleteKey(keyName); + } catch (IOException e) { + throw new Exception("Cannot delete encryption key: " + e.getMessage()); + } + + writeTestOutput("Encryption key deleted: '" + keyName + "'"); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java (working copy) @@ -29,18 +29,40 @@ SET(), RESET(), DFS(), + CRYPTO(true), ADD(), LIST(), RELOAD(), DELETE(), COMPILE(); + + public static boolean ONLY_FOR_TESTING = true; + private boolean usedOnlyForTesting; + + HiveCommand() { + this(false); + } + + HiveCommand(boolean onlyForTesting) { + this.usedOnlyForTesting = onlyForTesting; + } + + public boolean isOnlyForTesting() { + return this.usedOnlyForTesting; + } + private static final Set COMMANDS = new HashSet(); static { for (HiveCommand command : HiveCommand.values()) { COMMANDS.add(command.name()); } } + public static HiveCommand find(String[] command) { + return find(command, false); + } + + public static HiveCommand find(String[] command, boolean findOnlyForTesting) { if (null == command){ return null; } @@ -54,7 +76,13 @@ //special handling for SQL "delete from
where..." return null; } else if (COMMANDS.contains(cmd)) { - return HiveCommand.valueOf(cmd); + HiveCommand hiveCommand = HiveCommand.valueOf(cmd); + + if (findOnlyForTesting == hiveCommand.isOnlyForTesting()) { + return hiveCommand; + } + + return null; } } return null; Index: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (working copy) @@ -66,6 +66,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl; import org.apache.hadoop.hive.ql.util.DosToUnix; +import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -184,6 +185,11 @@ private SparkSession sparkSession; /** + * Gets information about HDFS encryption + */ + private HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; + + /** * Lineage state. */ LineageState ls; @@ -378,6 +384,23 @@ return txnAutoCommit; } + public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException { + if (hdfsEncryptionShim == null) { + try { + FileSystem fs = FileSystem.get(conf); + if ("hdfs".equals(fs.getUri().getScheme())) { + hdfsEncryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); + } else { + LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); + } + } catch (Exception e) { + throw new HiveException(e); + } + } + + return hdfsEncryptionShim; + } + /** * Singleton Session object per thread. * @@ -410,7 +433,6 @@ * when switching from one session to another. */ public static SessionState start(SessionState startSs) { - setCurrentSessionState(startSs); if (startSs.hiveHist == null){ Index: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (working copy) @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.CompactionRequest; @@ -262,7 +263,7 @@ private long sumDirSize(FileSystem fs, Path dir) throws IOException { long size = 0; - FileStatus[] buckets = fs.listStatus(dir); + FileStatus[] buckets = fs.listStatus(dir, FileUtils.HIDDEN_FILES_PATH_FILTER); for (int i = 0; i < buckets.length; i++) { size += buckets[i].getLen(); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java (working copy) @@ -42,7 +42,7 @@ extended = "date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or " + "'yyyy-MM-dd'.\n" + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" + " 30") + + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 30") @VectorizedExpressions({VectorUDFDayOfMonthLong.class, VectorUDFDayOfMonthString.class}) public class UDFDayOfMonth extends UDF { private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java (working copy) @@ -40,7 +40,7 @@ @Description(name = "month", value = "_FUNC_(date) - Returns the month of date", extended = "Example:\n" - + " > SELECT _FUNC_('2009-30-07') FROM src LIMIT 1;\n" + " 7") + + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 7") @VectorizedExpressions({VectorUDFMonthLong.class, VectorUDFMonthString.class}) public class UDFMonth extends UDF { private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java (working copy) @@ -42,7 +42,7 @@ extended = "date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or " + "'yyyy-MM-dd'.\n" + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" + " 2009") + + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 2009") @VectorizedExpressions({VectorUDFYearLong.class, VectorUDFYearString.class}) public class UDFYear extends UDF { private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java (working copy) @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; + +/** + * GenericUDFAddMonths. + * + * Add a number of months to the date. The time part of the string will be + * ignored. + * + */ +@Description(name = "add_months", + value = "_FUNC_(start_date, num_months) - Returns the date that is num_months after start_date.", + extended = "start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or" + + " 'yyyy-MM-dd'. num_months is a number. The time part of start_date is " + + "ignored.\n" + + "Example:\n " + " > SELECT _FUNC_('2009-08-31', 1) FROM src LIMIT 1;\n" + " '2009-09-30'") +public class GenericUDFAddMonths extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient TimestampConverter timestampConverter; + private transient Converter textConverter; + private transient Converter dateWritableConverter; + private transient Converter intWritableConverter; + private transient PrimitiveCategory inputType1; + private transient PrimitiveCategory inputType2; + private final Calendar calendar = Calendar.getInstance(); + private final Text output = new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 2) { + throw new UDFArgumentLengthException("add_months() requires 2 argument, got " + + arguments.length); + } + if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + + arguments[0].getTypeName() + " is passed. as first arguments"); + } + if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + + arguments[2].getTypeName() + " is passed. as second arguments"); + } + inputType1 = ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + switch (inputType1) { + case STRING: + case VARCHAR: + case CHAR: + inputType1 = PrimitiveCategory.STRING; + textConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + timestampConverter = new TimestampConverter((PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + dateWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + " ADD_MONTHS() only takes STRING/TIMESTAMP/DATEWRITABLE types as first argument, got " + + inputType1); + } + inputType2 = ((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory(); + if (inputType2 != PrimitiveCategory.INT) { + throw new UDFArgumentException(" ADD_MONTHS() only takes INT types as second argument, got " + + inputType2); + } + intWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[1], + PrimitiveObjectInspectorFactory.writableIntObjectInspector); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (arguments[0].get() == null) { + return null; + } + IntWritable toBeAdded = (IntWritable) intWritableConverter.convert(arguments[1].get()); + if (toBeAdded == null) { + return null; + } + Date date; + switch (inputType1) { + case STRING: + String dateString = textConverter.convert(arguments[0].get()).toString(); + try { + date = formatter.parse(dateString.toString()); + } catch (ParseException e) { + return null; + } + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) timestampConverter.convert(arguments[0].get())) + .getTimestamp(); + date = ts; + break; + case DATE: + DateWritable dw = (DateWritable) dateWritableConverter.convert(arguments[0].get()); + date = dw.get(); + break; + default: + throw new UDFArgumentException( + "ADD_MONTHS() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType1); + } + int numMonth = toBeAdded.get(); + addMonth(date, numMonth); + Date newDate = calendar.getTime(); + output.set(formatter.format(newDate)); + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("add_months("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + + protected Calendar addMonth(Date d, int numMonths) { + calendar.setTime(d); + + boolean lastDatOfMonth = isLastDayOfMonth(calendar); + + calendar.add(Calendar.MONTH, numMonths); + + if (lastDatOfMonth) { + int maxDd = calendar.getActualMaximum(Calendar.DAY_OF_MONTH); + calendar.set(Calendar.DAY_OF_MONTH, maxDd); + } + return calendar; + } + + protected boolean isLastDayOfMonth(Calendar cal) { + int maxDd = cal.getActualMaximum(Calendar.DAY_OF_MONTH); + int dd = cal.get(Calendar.DAY_OF_MONTH); + return dd == maxDd; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java (working copy) @@ -61,8 +61,8 @@ + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " + "ignored.\n" + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" - + " '2009-31-07'") + + " > SELECT _FUNC_('2009-07-30', 1) FROM src LIMIT 1;\n" + + " '2009-07-31'") @VectorizedExpressions({VectorUDFDateAddColScalar.class, VectorUDFDateAddScalarCol.class, VectorUDFDateAddColCol.class}) public class GenericUDFDateAdd extends GenericUDF { private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java (working copy) @@ -57,7 +57,7 @@ + "'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time parts are ignored." + "If date1 is earlier than date2, the result is negative.\n" + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', '2009-31-07') FROM src LIMIT 1;\n" + + " > SELECT _FUNC_('2009-07-30', '2009-07-31') FROM src LIMIT 1;\n" + " 1") @VectorizedExpressions({VectorUDFDateDiffColScalar.class, VectorUDFDateDiffColCol.class, VectorUDFDateDiffScalarCol.class}) public class GenericUDFDateDiff extends GenericUDF { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateSub.java (working copy) @@ -61,8 +61,8 @@ + " 'yyyy-MM-dd'. num_days is a number. The time part of start_date is " + "ignored.\n" + "Example:\n " - + " > SELECT _FUNC_('2009-30-07', 1) FROM src LIMIT 1;\n" - + " '2009-29-07'") + + " > SELECT _FUNC_('2009-07-30', 1) FROM src LIMIT 1;\n" + + " '2009-07-29'") @VectorizedExpressions({VectorUDFDateSubColScalar.class, VectorUDFDateSubScalarCol.class, VectorUDFDateSubColCol.class}) public class GenericUDFDateSub extends GenericUDF { private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java (working copy) @@ -19,11 +19,11 @@ package org.apache.hadoop.hive.ql.udf.generic; import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashSet; import org.apache.hadoop.hive.ql.exec.Description; @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IOUtils; /** * IN_FILE(str, filename) returns true if 'str' appears in the file specified @@ -106,29 +107,37 @@ arguments[0].get(), strObjectInspector).toString(); if (set == null) { - String fileName = (String)ObjectInspectorUtils.copyToStandardJavaObject( + String filePath = (String)ObjectInspectorUtils.copyToStandardJavaObject( arguments[1].get(), fileObjectInspector); - try { - load(new FileInputStream((new File(fileName)).getName())); - } catch (FileNotFoundException e) { - throw new HiveException(e); - } + loadFromFile(filePath); } - return Boolean.valueOf(set.contains(str)); + return set.contains(str); } - /** - * Load the file from an InputStream. - * @param is The InputStream contains the file data. - * @throws HiveException - */ - public void load(InputStream is) throws HiveException { - BufferedReader reader = - new BufferedReader(new InputStreamReader(is)); + private BufferedReader getReaderFor(String filePath) throws HiveException { + try { + Path fullFilePath = FileSystems.getDefault().getPath(filePath); + Path fileName = fullFilePath.getFileName(); + if (Files.exists(fileName)) { + return Files.newBufferedReader(fileName, Charset.defaultCharset()); + } + else + if (Files.exists(fullFilePath)) { + return Files.newBufferedReader(fullFilePath, Charset.defaultCharset()); + } + else { + throw new HiveException("Could not find \"" + fileName + "\" or \"" + fullFilePath + "\" in IN_FILE() UDF."); + } + } + catch(IOException exception) { + throw new HiveException(exception); + } + } + private void loadFromFile(String filePath) throws HiveException { set = new HashSet(); - + BufferedReader reader = getReaderFor(filePath); try { String line; while((line = reader.readLine()) != null) { @@ -137,9 +146,23 @@ } catch (Exception e) { throw new HiveException(e); } + finally { + IOUtils.closeStream(reader); + } } @Override + public void copyToNewInstance(Object newInstance) throws UDFArgumentException { + super.copyToNewInstance(newInstance); // Asserts the class invariant. (Same types.) + GenericUDFInFile that = (GenericUDFInFile)newInstance; + if (that != this) { + that.set = (this.set == null ? null : (HashSet)this.set.clone()); + that.strObjectInspector = this.strObjectInspector; + that.fileObjectInspector = this.fileObjectInspector; + } + } + + @Override public String getDisplayString(String[] children) { assert (children.length == 2); return "in_file(" + children[0] + ", " + children[1] + ")"; Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInitCap.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInitCap.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInitCap.java (working copy) @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.commons.lang.WordUtils; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringInitCap; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +/** + * UDFInitCap. + * + */ +@Description( + name = "initcap", + value = "_FUNC_(str) - Returns str, with the first letter of each word in uppercase," + + " all other letters in lowercase. Words are delimited by white space.", + extended = "Example:\n" + " > SELECT _FUNC_('tHe soap') FROM src LIMIT 1;\n" + " 'The Soap'") +@VectorizedExpressions({ StringInitCap.class }) +public class GenericUDFInitCap extends GenericUDF { + private transient PrimitiveObjectInspector argumentOI; + private transient StringConverter stringConverter; + private transient PrimitiveCategory returnType = PrimitiveCategory.STRING; + private transient GenericUDFUtils.StringHelper returnHelper; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException("INITCAP requires 1 argument, got " + arguments.length); + } + if (arguments[0].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentException("INITCAP only takes primitive types, got " + + argumentOI.getTypeName()); + } + argumentOI = (PrimitiveObjectInspector) arguments[0]; + stringConverter = new PrimitiveObjectInspectorConverter.StringConverter(argumentOI); + PrimitiveCategory inputType = argumentOI.getPrimitiveCategory(); + ObjectInspector outputOI = null; + BaseCharTypeInfo typeInfo; + switch (inputType) { + case CHAR: + // return type should have same length as the input. + returnType = inputType; + typeInfo = TypeInfoFactory.getCharTypeInfo(GenericUDFUtils.StringHelper + .getFixedStringSizeForType(argumentOI)); + outputOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(typeInfo); + break; + case VARCHAR: + // return type should have same length as the input. + returnType = inputType; + typeInfo = TypeInfoFactory.getVarcharTypeInfo(GenericUDFUtils.StringHelper + .getFixedStringSizeForType(argumentOI)); + outputOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(typeInfo); + break; + default: + returnType = PrimitiveCategory.STRING; + outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + break; + } + returnHelper = new GenericUDFUtils.StringHelper(returnType); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + String val = null; + if (arguments[0] != null) { + val = (String) stringConverter.convert(arguments[0].get()); + } + if (val == null) { + return null; + } + + val = WordUtils.capitalizeFully(val); + + return returnHelper.setReturnValue(val); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("initcap("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(","); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java (working copy) @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +/** + * GenericUDFLastDay. + * + * Returns the last day of the month which the date belongs to. + * The time part of the date will be ignored. + * + */ +@Description(name = "last_day", + value = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", + extended = "date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'." + + " The time part of date is ignored.\n" + + "Example:\n " + " > SELECT _FUNC_('2009-01-12') FROM src LIMIT 1;\n" + " '2009-01-31'") +public class GenericUDFLastDay extends GenericUDF { + private transient SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + private transient TimestampConverter timestampConverter; + private transient Converter textConverter; + private transient Converter dateWritableConverter; + private transient PrimitiveCategory inputType1; + private final Calendar calendar = Calendar.getInstance(); + private final Text output = new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException("last_day() requires 1 argument, got " + + arguments.length); + } + if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + + arguments[0].getTypeName() + " is passed. as first arguments"); + } + inputType1 = ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory(); + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + switch (inputType1) { + case STRING: + case VARCHAR: + case CHAR: + inputType1 = PrimitiveCategory.STRING; + textConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + break; + case TIMESTAMP: + timestampConverter = new TimestampConverter((PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); + break; + case DATE: + dateWritableConverter = ObjectInspectorConverters.getConverter( + (PrimitiveObjectInspector) arguments[0], + PrimitiveObjectInspectorFactory.writableDateObjectInspector); + break; + default: + throw new UDFArgumentException( + " LAST_DAY() only takes STRING/TIMESTAMP/DATEWRITABLE types as first argument, got " + + inputType1); + } + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (arguments[0].get() == null) { + return null; + } + Date date; + switch (inputType1) { + case STRING: + String dateString = textConverter.convert(arguments[0].get()).toString(); + try { + date = formatter.parse(dateString.toString()); + } catch (ParseException e) { + return null; + } + lastDay(date); + break; + case TIMESTAMP: + Timestamp ts = ((TimestampWritable) timestampConverter.convert(arguments[0].get())) + .getTimestamp(); + date = ts; + lastDay(date); + break; + case DATE: + DateWritable dw = (DateWritable) dateWritableConverter.convert(arguments[0].get()); + date = dw.get(); + lastDay(date); + break; + default: + throw new UDFArgumentException( + "LAST_DAY() only takes STRING/TIMESTAMP/DATEWRITABLE types, got " + inputType1); + } + Date newDate = calendar.getTime(); + output.set(formatter.format(newDate)); + return output; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("last_day("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + + protected Calendar lastDay(Date d) { + calendar.setTime(d); + int maxDd = calendar.getActualMaximum(Calendar.DAY_OF_MONTH); + calendar.set(Calendar.DAY_OF_MONTH, maxDd); + return calendar; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java (working copy) @@ -51,7 +51,7 @@ @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if (arguments.length < 0) { + if (arguments.length != 1) { throw new UDFArgumentLengthException( "LOWER requires 1 argument, got " + arguments.length); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java (revision 1653459) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java (working copy) @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; /** * UDFUpper. @@ -52,7 +51,7 @@ @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if (arguments.length < 0) { + if (arguments.length != 1) { throw new UDFArgumentLengthException( "UPPER requires 1 argument, got " + arguments.length); } Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java (working copy) @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.AfterClass; +import org.junit.Test; + +public class TestHooks { + + @BeforeClass + public static void onetimeSetup() throws Exception { + HiveConf conf = new HiveConf(TestHooks.class); + Driver driver = createDriver(conf); + int ret = driver.run("create table t1(i int)").getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + + @AfterClass + public static void onetimeTeardown() throws Exception { + HiveConf conf = new HiveConf(TestHooks.class); + Driver driver = createDriver(conf); + driver.run("drop table t1"); + } + + @Before + public void setup() { + } + + @Test + public void testQueryRedactor() throws Exception { + HiveConf conf = new HiveConf(TestHooks.class); + HiveConf.setVar(conf, HiveConf.ConfVars.QUERYREDACTORHOOKS, + SimpleQueryRedactor.class.getName()); + Driver driver = createDriver(conf); + int ret = driver.compile("select 'XXX' from t1"); + assertEquals("Checking command success", 0, ret); + assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING)); + } + + public static class SimpleQueryRedactor extends Redactor { + public String redactQuery(String query) { + return query.replaceAll("XXX", "AAA"); + } + } + + private static Driver createDriver(HiveConf conf) { + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + SessionState.start(conf); + Driver driver = new Driver(conf); + driver.init(); + return driver; + } + +} Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (revision 1653459) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (working copy) @@ -101,7 +101,7 @@ } /** - * Test combine symlink text input file. Two input dir, and each contails one + * Test combine symlink text input file. Two input dir, and each contains one * file, and then create one symlink file containing these 2 files. Normally * without combine, it will return at least 2 splits */ @@ -166,7 +166,11 @@ } String cmd = "select key*1 from " + tblName; - drv.compile(cmd); + ecode = drv.compile(cmd); + if (ecode != 0) { + throw new Exception("Select compile: " + cmd + + " failed with exit code= " + ecode); + } //create scratch dir Context ctx = new Context(newJob); Index: ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java (revision 1653459) +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java (working copy) @@ -18,71 +18,83 @@ package org.apache.hadoop.hive.ql.lockmgr.zookeeper; -import static org.mockito.Mockito.*; - -import java.util.Collections; - import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryOneTime; +import org.apache.curator.test.TestingServer; import org.junit.Assert; import org.junit.Before; +import org.junit.After; import org.junit.Test; -import com.google.common.base.Joiner; - public class TestZookeeperLockManager { - private static final Joiner SLASH = Joiner.on("/"); - private static final String PARENT = "hive"; - private static final String TABLE = "t1"; - private static final String PARENT_LOCK_PATH = SLASH.join("", PARENT, TABLE); - private static final String TABLE_LOCK_PATH = SLASH.join("", PARENT, TABLE, "00001"); private HiveConf conf; - private ZooKeeper zooKeeper; + private TestingServer server; + private CuratorFramework client; private HiveLockObject hiveLock; private ZooKeeperHiveLock zLock; + private HiveLockObjectData lockObjData; + private static final String PARENT = "hive"; + private static final String TABLE = "t1"; + private static final String PARENT_LOCK_PATH = "/hive/t1"; + private static final String TABLE_LOCK_PATH = "/hive/t1/00001"; @Before public void setup() { conf = new HiveConf(); - zooKeeper = mock(ZooKeeper.class); - hiveLock = mock(HiveLockObject.class); - when(hiveLock.getName()).thenReturn(TABLE); + lockObjData = new HiveLockObjectData("1", "10", "SHARED", "show tables"); + hiveLock = new HiveLockObject(TABLE, lockObjData); zLock = new ZooKeeperHiveLock(TABLE_LOCK_PATH, hiveLock, HiveLockMode.SHARED); + + while (server == null) + { + try { + server = new TestingServer(); + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder(); + client = builder.connectString(server.getConnectString()).retryPolicy(new RetryOneTime(1)).build(); + client.start(); + } catch (Exception e) { + System.err.println("Getting bind exception - retrying to allocate server"); + server = null; + } + } } - @Test - public void testDeleteNoChildren() throws Exception { - ZooKeeperHiveLockManager.unlockPrimitive(conf, zooKeeper, zLock, PARENT); - verify(zooKeeper).delete(TABLE_LOCK_PATH, -1); - verify(zooKeeper).getChildren(PARENT_LOCK_PATH, false); - verify(zooKeeper).delete(PARENT_LOCK_PATH, -1); - verifyNoMoreInteractions(zooKeeper); + @After + public void teardown() throws Exception + { + client.close(); + server.close(); + server = null; } - /** - * Tests two threads racing to delete PARENT_LOCK_PATH - */ + @Test - public void testDeleteNoChildrenNodeDoesNotExist() throws Exception { - doThrow(new KeeperException.NoNodeException()).when(zooKeeper).delete(PARENT_LOCK_PATH, -1); - ZooKeeperHiveLockManager.unlockPrimitive(conf, zooKeeper, zLock, PARENT); - verify(zooKeeper).delete(TABLE_LOCK_PATH, -1); - verify(zooKeeper).getChildren(PARENT_LOCK_PATH, false); - verify(zooKeeper).delete(PARENT_LOCK_PATH, -1); - verifyNoMoreInteractions(zooKeeper); + public void testDeleteNoChildren() throws Exception + { + client.create().creatingParentsIfNeeded().forPath(TABLE_LOCK_PATH, lockObjData.toString().getBytes()); + byte[] data = client.getData().forPath(TABLE_LOCK_PATH); + Assert.assertArrayEquals(lockObjData.toString().getBytes(), data); + ZooKeeperHiveLockManager.unlockPrimitive(zLock, PARENT, client); + try { + data = client.getData().forPath(TABLE_LOCK_PATH); + Assert.fail(); + } catch (Exception e) { + Assert.assertEquals( e instanceof KeeperException.NoNodeException, true); + } + try { + data = client.getData().forPath(PARENT_LOCK_PATH); + Assert.fail(); + } catch (Exception e) { + Assert.assertEquals( e instanceof KeeperException.NoNodeException, true); + } } - @Test - public void testDeleteWithChildren() throws Exception { - when(zooKeeper.getChildren(PARENT_LOCK_PATH, false)).thenReturn(Collections.singletonList("somechild")); - ZooKeeperHiveLockManager.unlockPrimitive(conf, zooKeeper, zLock, PARENT); - verify(zooKeeper).delete(TABLE_LOCK_PATH, -1); - verify(zooKeeper).getChildren(PARENT_LOCK_PATH, false); - verifyNoMoreInteractions(zooKeeper); - } @Test public void testGetQuorumServers() { @@ -99,3 +111,4 @@ Assert.assertEquals("node1:5666,node2:9999,node3:9999", ZooKeeperHiveHelper.getQuorumServers(conf)); } } + Index: ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java (revision 1653459) +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java (working copy) @@ -95,7 +95,7 @@ rs = new ReduceSinkOperator(); rs.setConf(new ReduceSinkDesc()); ts = new TableScanOperator(); - ts.setConf(new TableScanDesc()); + ts.setConf(new TableScanDesc(null)); ts.getChildOperators().add(rs); rs.getParentOperators().add(ts); rs.getChildOperators().add(fs); Index: ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java (revision 1653459) +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java (working copy) @@ -193,7 +193,7 @@ } @Test public void testInsertIntoTableAsSelectFromNamedVirtTable() throws ParseException { - ASTNode ast = parse("insert into table page_view select a,b as c from (values (1,2),(3,4)) as VC(a,b) where b = 9"); + ASTNode ast = parse("insert into page_view select a,b as c from (values (1,2),(3,4)) as VC(a,b) where b = 9"); Assert.assertEquals("AST doesn't match", "(TOK_QUERY " + "(TOK_FROM " + @@ -209,7 +209,7 @@ } @Test public void testInsertIntoTableFromAnonymousTable1Row() throws ParseException { - ASTNode ast = parse("insert into table page_view values(1,2)"); + ASTNode ast = parse("insert into page_view values(1,2)"); Assert.assertEquals("AST doesn't match", "(TOK_QUERY " + "(TOK_FROM " + @@ -232,5 +232,16 @@ "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " + "(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))", ast.toStringTree()); + //same query as above less the "table" keyword KW_TABLE + ast = parse("insert into page_view values(-1,2),(3,+4)"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF TOK_ANONYMOUS) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW (- 1) 2) (TOK_VALUE_ROW 3 (+ 4))))) " + + "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " + + "(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))", + ast.toStringTree()); } } Index: ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java (revision 1653459) +++ ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java (working copy) @@ -20,15 +20,17 @@ import java.sql.SQLException; -import junit.framework.Assert; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class TestCommandProcessorFactory { + private final String[] testOnlyCommands = new String[]{"crypto"}; + private HiveConf conf; @Before @@ -38,27 +40,39 @@ @Test public void testInvalidCommands() throws Exception { - Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null, conf)); - Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); - Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); - Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Null should have returned null", + CommandProcessorFactory.getForHiveCommand(null, conf)); + Assert.assertNull("Blank should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); + Assert.assertNull("Set role should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); + Assert.assertNull("SQL should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Test only command should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"CRYPTO --listZones"}, conf)); } + @Test public void testAvailableCommands() throws Exception { + enableTestOnlyCmd(conf); SessionState.start(conf); + for (HiveCommand command : HiveCommand.values()) { String cmd = command.name(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + String cmdInLowerCase = cmd.toLowerCase(); + Assert.assertNotNull("Cmd " + cmd + " not return null", + CommandProcessorFactory + .getForHiveCommandInternal(new String[]{cmd}, conf, command.isOnlyForTesting())); + Assert.assertNotNull("Cmd " + cmd + " not return null", + CommandProcessorFactory.getForHiveCommandInternal( + new String[]{cmdInLowerCase}, conf, command.isOnlyForTesting())); } - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name().toLowerCase(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); - } conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), ""); for (HiveCommand command : HiveCommand.values()) { String cmd = command.name(); try { - CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf); + CommandProcessorFactory + .getForHiveCommandInternal(new String[]{cmd}, conf, command.isOnlyForTesting()); Assert.fail("Expected SQLException for " + cmd + " as available commands is empty"); } catch (SQLException e) { Assert.assertEquals("Insufficient privileges to execute " + cmd, e.getMessage()); @@ -66,4 +80,13 @@ } } } + + private void enableTestOnlyCmd(HiveConf conf){ + StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST)); + for(String c : testOnlyCommands){ + securityCMDs.append(","); + securityCMDs.append(c); + } + conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString()); + } } Index: ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFAddMonths.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFAddMonths.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFAddMonths.java (working copy) @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; + +import junit.framework.TestCase; + +public class TestGenericUDFAddMonths extends TestCase { + + public void testAddMonths() throws HiveException { + GenericUDFAddMonths udf = new GenericUDFAddMonths(); + ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + ObjectInspector[] arguments = { valueOI0, valueOI1 }; + + udf.initialize(arguments); + runAndVerify("2014-01-14", 1, "2014-02-14", udf); + runAndVerify("2014-01-31", 1, "2014-02-28", udf); + runAndVerify("2014-02-28", -1, "2014-01-31", udf); + runAndVerify("2014-02-28", 2, "2014-04-30", udf); + runAndVerify("2014-04-30", -2, "2014-02-28", udf); + runAndVerify("2015-02-28", 12, "2016-02-29", udf); + runAndVerify("2016-02-29", -12, "2015-02-28", udf); + runAndVerify("2016-01-29", 1, "2016-02-29", udf); + runAndVerify("2016-02-29", -1, "2016-01-31", udf); + } + + private void runAndVerify(String str, int months, String expResult, GenericUDF udf) + throws HiveException { + DeferredObject valueObj0 = new DeferredJavaObject(new Text(str)); + DeferredObject valueObj1 = new DeferredJavaObject(new IntWritable(months)); + DeferredObject[] args = { valueObj0, valueObj1 }; + Text output = (Text) udf.evaluate(args); + assertEquals("add_months() test ", expResult, output.toString()); + } +} Index: ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitCap.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitCap.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitCap.java (working copy) @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +import junit.framework.TestCase; + +public class TestGenericUDFInitCap extends TestCase { + + public void testInitCap() throws HiveException { + GenericUDFInitCap udf = new GenericUDFInitCap(); + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + ObjectInspector[] arguments = { valueOI }; + + udf.initialize(arguments); + runAndVerify(" hello World! ", " Hello World! ", udf); + runAndVerify("helLo world! ", "Hello World! ", udf); + runAndVerify("hello world!", "Hello World!", udf); + runAndVerify(" \t\r\nhello \t\r\nworld! \t\r\n", " \t\r\nHello \t\r\nWorld! \t\r\n", udf); + runAndVerify("Hello World!", "Hello World!", udf); + runAndVerify(" ", " ", udf); + } + + private void runAndVerify(String str, String expResult, GenericUDF udf) throws HiveException { + DeferredObject valueObj = new DeferredJavaObject(new Text(str)); + DeferredObject[] args = { valueObj }; + Text output = (Text) udf.evaluate(args); + assertEquals("initcap() test ", expResult, output.toString()); + } +} Index: ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java (working copy) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; + +import junit.framework.TestCase; + +public class TestGenericUDFLastDay extends TestCase { + + public void testLastDay() throws HiveException { + GenericUDFLastDay udf = new GenericUDFLastDay(); + ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + ObjectInspector[] arguments = { valueOI0 }; + + udf.initialize(arguments); + runAndVerify("2014-01-01", "2014-01-31", udf); + runAndVerify("2014-01-14", "2014-01-31", udf); + runAndVerify("2014-01-31", "2014-01-31", udf); + runAndVerify("2014-02-02", "2014-02-28", udf); + runAndVerify("2014-02-28", "2014-02-28", udf); + runAndVerify("2016-02-03", "2016-02-29", udf); + runAndVerify("2016-02-28", "2016-02-29", udf); + runAndVerify("2016-02-29", "2016-02-29", udf); + + runAndVerify("2014-01-01 10:30:45", "2014-01-31", udf); + runAndVerify("2014-01-14 10:30:45", "2014-01-31", udf); + runAndVerify("2014-01-31 10:30:45", "2014-01-31", udf); + runAndVerify("2014-02-02 10:30:45", "2014-02-28", udf); + runAndVerify("2014-02-28 10:30:45", "2014-02-28", udf); + runAndVerify("2016-02-03 10:30:45", "2016-02-29", udf); + runAndVerify("2016-02-28 10:30:45", "2016-02-29", udf); + runAndVerify("2016-02-29 10:30:45", "2016-02-29", udf); + } + + private void runAndVerify(String str, String expResult, GenericUDF udf) + throws HiveException { + DeferredObject valueObj0 = new DeferredJavaObject(new Text(str)); + DeferredObject[] args = { valueObj0 }; + Text output = (Text) udf.evaluate(args); + assertEquals("last_day() test ", expResult, output.toString()); + } +} Index: ql/src/test/queries/clientnegative/selectDistinctStarNeg_1.q =================================================================== --- ql/src/test/queries/clientnegative/selectDistinctStarNeg_1.q (revision 0) +++ ql/src/test/queries/clientnegative/selectDistinctStarNeg_1.q (working copy) @@ -0,0 +1,4 @@ +-- Duplicate column name: key + +drop view if exists v; +create view v as select distinct * from src join src1 on src.key=src1.key; \ No newline at end of file Index: ql/src/test/queries/clientnegative/selectDistinctStarNeg_2.q =================================================================== --- ql/src/test/queries/clientnegative/selectDistinctStarNeg_2.q (revision 0) +++ ql/src/test/queries/clientnegative/selectDistinctStarNeg_2.q (working copy) @@ -0,0 +1,3 @@ +-- SELECT DISTINCT and GROUP BY can not be in the same query. Error encountered near token ‘key’ + +select distinct * from src group by key; \ No newline at end of file Index: ql/src/test/queries/clientpositive/auto_join21.q =================================================================== --- ql/src/test/queries/clientpositive/auto_join21.q (revision 1653459) +++ ql/src/test/queries/clientpositive/auto_join21.q (working copy) @@ -1,4 +1,7 @@ set hive.auto.convert.join = true; + +-- SORT_QUERY_RESULTS + explain SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value; Index: ql/src/test/queries/clientpositive/auto_join23.q =================================================================== --- ql/src/test/queries/clientpositive/auto_join23.q (revision 1653459) +++ ql/src/test/queries/clientpositive/auto_join23.q (working copy) @@ -1,5 +1,7 @@ set hive.auto.convert.join = true; +-- SORT_QUERY_RESULTS + explain SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value; Index: ql/src/test/queries/clientpositive/constprog_partitioner.q =================================================================== --- ql/src/test/queries/clientpositive/constprog_partitioner.q (revision 1653459) +++ ql/src/test/queries/clientpositive/constprog_partitioner.q (working copy) @@ -9,3 +9,16 @@ SELECT src1.key, src1.key + 1, src2.value FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100; + +EXPLAIN +SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +; + +SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +; Index: ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q (working copy) @@ -0,0 +1,57 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +-- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable; +drop table IF EXISTS unencryptedTable; + +create table encryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc + LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encryptedTable' TBLPROPERTIES ('transactional'='true'); +CRYPTO CREATE_KEY --keyName key_1 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/encryptedTable; + +create table unencryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + +-- insert encrypted table from values +explain extended insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502'); + +insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502'); + +select * from encryptedTable order by key; + +-- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition (key) + select value, key limit 2; + +from src +insert into table encryptedTable partition (key) + select value, key limit 2; + +select * from encryptedTable order by key; + +-- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition (key) + select value, key; + +from encryptedTable +insert into table unencryptedTable partition (key) + select value, key; + +select * from unencryptedTable order by key; + +-- clean up +drop table encryptedTable; +CRYPTO DELETE_KEY --keyName key_1; +drop table unencryptedTable; \ No newline at end of file Index: ql/src/test/queries/clientpositive/encryption_insert_partition_static.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_insert_partition_static.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_insert_partition_static.q (working copy) @@ -0,0 +1,62 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +-- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable; +drop table IF EXISTS unencryptedTable; + +create table encryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc + LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encryptedTable' TBLPROPERTIES ('transactional'='true'); +CRYPTO CREATE_KEY --keyName key_1 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/encryptedTable; + +create table unencryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + +-- insert encrypted table from values +explain extended insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502'); + +insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502'); + +select * from encryptedTable order by key; + +-- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2; + +from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2; + +select * from encryptedTable order by key; + +-- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value; + +from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value; + +select * from unencryptedTable order by key; + +-- clean up +drop table encryptedTable; +CRYPTO DELETE_KEY --keyName key_1; +drop table unencryptedTable; \ No newline at end of file Index: ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q (working copy) @@ -0,0 +1,15 @@ +--SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table; +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; + +SELECT * FROM encrypted_table; + +EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key; + +drop table encrypted_table; +CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file Index: ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q (working copy) @@ -0,0 +1,24 @@ +--SORT_QUERY_RESULTS + +-- Java JCE must be installed in order to hava a key length of 256 bits +DROP TABLE IF EXISTS table_key_1; +CREATE TABLE table_key_1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_1'; +CRYPTO CREATE_KEY --keyName key_1 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_1; + +DROP TABLE IF EXISTS table_key_2; +CREATE TABLE table_key_2 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_2'; +CRYPTO CREATE_KEY --keyName key_2 --bitLength 256; +CRYPTO CREATE_ZONE --keyName key_2 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_2; + +INSERT OVERWRITE TABLE table_key_1 SELECT * FROM src; +INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src; + +EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); +SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); + +DROP TABLE table_key_1; +DROP TABLE table_key_2; + +CRYPTO DELETE_KEY --keyName key_1; +CRYPTO DELETE_KEY --keyName key_2; \ No newline at end of file Index: ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q (working copy) @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS encrypted_table; + +CREATE TABLE encrypted_table (key STRING, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_table'; + +-- Create encryption key and zone; +crypto create_key --keyName key1; +crypto create_zone --keyName key1 --path ${hiveconf:hive.metastore.warehouse.dir}/encrypted_table; + +-- Test loading data from the local filesystem; +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE encrypted_table; +SELECT * FROM encrypted_table; + +-- Test loading data from the hdfs filesystem; +dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/kv1.txt; +LOAD DATA INPATH '/tmp/kv1.txt' OVERWRITE INTO TABLE encrypted_table; +SELECT * FROM encrypted_table; + +DROP TABLE encrypted_table; + +crypto delete_key --keyName key1; \ No newline at end of file Index: ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q (working copy) @@ -0,0 +1,16 @@ +-- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table; +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; + +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE encrypted_table; + +dfs -chmod -R 555 ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +SELECT count(*) FROM encrypted_table; + +drop table encrypted_table; +CRYPTO DELETE_KEY --keyName key_128; Index: ql/src/test/queries/clientpositive/encryption_select_read_only_unencrypted_tbl.q =================================================================== --- ql/src/test/queries/clientpositive/encryption_select_read_only_unencrypted_tbl.q (revision 0) +++ ql/src/test/queries/clientpositive/encryption_select_read_only_unencrypted_tbl.q (working copy) @@ -0,0 +1,12 @@ +-- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS unencrypted_table; +CREATE TABLE unencrypted_table(key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/unencrypted_table'; + +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE unencrypted_table; + +dfs -chmod -R 555 ${hiveconf:hive.metastore.warehouse.dir}/default/unencrypted_table; + +SELECT count(*) FROM unencrypted_table; + +drop table unencrypted_table; \ No newline at end of file Index: ql/src/test/queries/clientpositive/groupby2.q =================================================================== --- ql/src/test/queries/clientpositive/groupby2.q (revision 1653459) +++ ql/src/test/queries/clientpositive/groupby2.q (working copy) @@ -11,4 +11,6 @@ FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1); +-- SORT_QUERY_RESULTS + SELECT dest_g2.* FROM dest_g2; Index: ql/src/test/queries/clientpositive/groupby_grouping_window.q =================================================================== --- ql/src/test/queries/clientpositive/groupby_grouping_window.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby_grouping_window.q (working copy) @@ -0,0 +1,15 @@ +create table t(category int, live int, comments int); +insert into table t select key, 0, 2 from src tablesample(3 rows); + +explain +select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0; + +select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0; Index: ql/src/test/queries/clientpositive/groupby_multi_single_reducer2.q =================================================================== --- ql/src/test/queries/clientpositive/groupby_multi_single_reducer2.q (revision 1653459) +++ ql/src/test/queries/clientpositive/groupby_multi_single_reducer2.q (working copy) @@ -3,6 +3,8 @@ CREATE TABLE dest_g2(key STRING, c1 INT) STORED AS TEXTFILE; CREATE TABLE dest_g3(key STRING, c1 INT, c2 INT) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + EXPLAIN FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) Index: ql/src/test/queries/clientpositive/groupby_ppr.q =================================================================== --- ql/src/test/queries/clientpositive/groupby_ppr.q (revision 1653459) +++ ql/src/test/queries/clientpositive/groupby_ppr.q (working copy) @@ -1,6 +1,8 @@ set hive.map.aggr=false; set hive.groupby.skewindata=false; +-- SORT_QUERY_RESULTS + CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; EXPLAIN EXTENDED Index: ql/src/test/queries/clientpositive/input14.q =================================================================== --- ql/src/test/queries/clientpositive/input14.q (revision 1653459) +++ ql/src/test/queries/clientpositive/input14.q (working copy) @@ -17,4 +17,6 @@ ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100; +-- SORT_QUERY_RESULTS + SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/input17.q =================================================================== --- ql/src/test/queries/clientpositive/input17.q (revision 1653459) +++ ql/src/test/queries/clientpositive/input17.q (working copy) @@ -17,4 +17,6 @@ ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; +-- SORT_QUERY_RESULTS + SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/input18.q =================================================================== --- ql/src/test/queries/clientpositive/input18.q (revision 1653459) +++ ql/src/test/queries/clientpositive/input18.q (working copy) @@ -17,4 +17,6 @@ ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100; +-- SORT_QUERY_RESULTS + SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/input_part2.q =================================================================== --- ql/src/test/queries/clientpositive/input_part2.q (revision 1653459) +++ ql/src/test/queries/clientpositive/input_part2.q (working copy) @@ -1,6 +1,8 @@ CREATE TABLE dest1(key INT, value STRING, hr STRING, ds STRING) STORED AS TEXTFILE; CREATE TABLE dest2(key INT, value STRING, hr STRING, ds STRING) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' Index: ql/src/test/queries/clientpositive/join0.q =================================================================== --- ql/src/test/queries/clientpositive/join0.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join0.q (working copy) @@ -1,4 +1,5 @@ -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, Index: ql/src/test/queries/clientpositive/join15.q =================================================================== --- ql/src/test/queries/clientpositive/join15.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join15.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key) SORT BY src1.key, src1.value, src2.key, src2.value; Index: ql/src/test/queries/clientpositive/join18.q =================================================================== --- ql/src/test/queries/clientpositive/join18.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join18.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT a.key, a.value, b.key, b.value FROM Index: ql/src/test/queries/clientpositive/join20.q =================================================================== --- ql/src/test/queries/clientpositive/join20.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join20.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value; Index: ql/src/test/queries/clientpositive/join21.q =================================================================== --- ql/src/test/queries/clientpositive/join21.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join21.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value; Index: ql/src/test/queries/clientpositive/join23.q =================================================================== --- ql/src/test/queries/clientpositive/join23.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join23.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value; Index: ql/src/test/queries/clientpositive/join6.q =================================================================== --- ql/src/test/queries/clientpositive/join6.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join6.q (working copy) @@ -1,5 +1,7 @@ CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + EXPLAIN FROM ( FROM Index: ql/src/test/queries/clientpositive/join7.q =================================================================== --- ql/src/test/queries/clientpositive/join7.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join7.q (working copy) @@ -1,5 +1,7 @@ CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + EXPLAIN FROM ( FROM Index: ql/src/test/queries/clientpositive/join_array.q =================================================================== --- ql/src/test/queries/clientpositive/join_array.q (revision 1653459) +++ ql/src/test/queries/clientpositive/join_array.q (working copy) @@ -4,6 +4,8 @@ load data local inpath '../../data/files/tiny_a.txt' into table tinyA; load data local inpath '../../data/files/tiny_b.txt' into table tinyB; +-- SORT_QUERY_RESULTS + select * from tinyA; select * from tinyB; Index: ql/src/test/queries/clientpositive/limit_partition_metadataonly.q =================================================================== --- ql/src/test/queries/clientpositive/limit_partition_metadataonly.q (revision 1653459) +++ ql/src/test/queries/clientpositive/limit_partition_metadataonly.q (working copy) @@ -1,5 +1,7 @@ set hive.limit.query.max.table.partition=1; +-- SORT_QUERY_RESULTS + explain select ds from srcpart where hr=11 and ds='2008-04-08'; select ds from srcpart where hr=11 and ds='2008-04-08'; Index: ql/src/test/queries/clientpositive/mapjoin_decimal.q =================================================================== --- ql/src/test/queries/clientpositive/mapjoin_decimal.q (revision 1653459) +++ ql/src/test/queries/clientpositive/mapjoin_decimal.q (working copy) @@ -29,14 +29,9 @@ select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec); set hive.mapjoin.optimized.hashtable=false; -set hive.mapjoin.optimized.keys=false; select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec); -set hive.mapjoin.optimized.keys=true; - -select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec); - set hive.mapjoin.optimized.hashtable=true; select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec); Index: ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q =================================================================== --- ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q (revision 1653459) +++ ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q (working copy) @@ -1,4 +1,7 @@ set hive.auto.convert.join = false; + +-- SORT_QUERY_RESULTS + --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition SELECT * FROM src1 Index: ql/src/test/queries/clientpositive/mapjoin_test_outer.q =================================================================== --- ql/src/test/queries/clientpositive/mapjoin_test_outer.q (revision 1653459) +++ ql/src/test/queries/clientpositive/mapjoin_test_outer.q (working copy) @@ -1,4 +1,7 @@ set hive.auto.convert.join = false; + +-- SORT_QUERY_RESULTS + --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition create table dest_1 (key STRING, value STRING) stored as textfile; Index: ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q =================================================================== --- ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q (revision 1653459) +++ ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q (working copy) @@ -7,9 +7,13 @@ explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10; explain select cast(key as int) * 10, upper(value) from src limit 10; +-- Scans without limit (should be Fetch task now) +explain select concat(key, value) from src; set hive.fetch.task.conversion.threshold=100; -- from HIVE-7397, limit + partition pruning filter explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10; explain select cast(key as int) * 10, upper(value) from src limit 10; +-- Scans without limit (should not be Fetch task now) +explain select concat(key, value) from src; Index: ql/src/test/queries/clientpositive/ppd_transform.q =================================================================== --- ql/src/test/queries/clientpositive/ppd_transform.q (revision 1653459) +++ ql/src/test/queries/clientpositive/ppd_transform.q (working copy) @@ -2,6 +2,8 @@ set hive.ppd.remove.duplicatefilters=false; set hive.entity.capture.transform=true; +-- SORT_QUERY_RESULTS + EXPLAIN FROM ( FROM src Index: ql/src/test/queries/clientpositive/ptf_matchpath.q =================================================================== --- ql/src/test/queries/clientpositive/ptf_matchpath.q (revision 1653459) +++ ql/src/test/queries/clientpositive/ptf_matchpath.q (working copy) @@ -12,6 +12,8 @@ LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny; +-- SORT_QUERY_RESULTS + -- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on @@ -43,4 +45,4 @@ arg2('LATE'), arg3(arr_delay > 15), arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ); - \ No newline at end of file + Index: ql/src/test/queries/clientpositive/ptf_rcfile.q =================================================================== --- ql/src/test/queries/clientpositive/ptf_rcfile.q (revision 1653459) +++ ql/src/test/queries/clientpositive/ptf_rcfile.q (working copy) @@ -14,6 +14,8 @@ LOAD DATA LOCAL INPATH '../../data/files/part.rc' overwrite into table part_rc; +-- SORT_QUERY_RESULTS + -- testWindowingPTFWithPartRC select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, Index: ql/src/test/queries/clientpositive/ptf_register_tblfn.q =================================================================== --- ql/src/test/queries/clientpositive/ptf_register_tblfn.q (revision 1653459) +++ ql/src/test/queries/clientpositive/ptf_register_tblfn.q (working copy) @@ -14,6 +14,7 @@ create temporary function matchpathtest as 'org.apache.hadoop.hive.ql.udf.ptf.MatchPath$MatchPathResolver'; +-- SORT_QUERY_RESULTS -- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath Index: ql/src/test/queries/clientpositive/ptf_seqfile.q =================================================================== --- ql/src/test/queries/clientpositive/ptf_seqfile.q (revision 1653459) +++ ql/src/test/queries/clientpositive/ptf_seqfile.q (working copy) @@ -14,6 +14,8 @@ LOAD DATA LOCAL INPATH '../../data/files/part.seq' overwrite into table part_seq; +-- SORT_QUERY_RESULTS + -- testWindowingPTFWithPartSeqFile select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, Index: ql/src/test/queries/clientpositive/sample3.q =================================================================== --- ql/src/test/queries/clientpositive/sample3.q (revision 1653459) +++ ql/src/test/queries/clientpositive/sample3.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + -- no input pruning, sample filter EXPLAIN SELECT s.key Index: ql/src/test/queries/clientpositive/sample5.q =================================================================== --- ql/src/test/queries/clientpositive/sample5.q (revision 1653459) +++ ql/src/test/queries/clientpositive/sample5.q (working copy) @@ -1,5 +1,7 @@ CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + -- no input pruning, sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test Index: ql/src/test/queries/clientpositive/scriptfile1.q =================================================================== --- ql/src/test/queries/clientpositive/scriptfile1.q (revision 1653459) +++ ql/src/test/queries/clientpositive/scriptfile1.q (working copy) @@ -1,5 +1,7 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +-- SORT_QUERY_RESULTS + -- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING); Index: ql/src/test/queries/clientpositive/selectDistinctStar.q =================================================================== --- ql/src/test/queries/clientpositive/selectDistinctStar.q (revision 0) +++ ql/src/test/queries/clientpositive/selectDistinctStar.q (working copy) @@ -0,0 +1,101 @@ +explain select distinct src.* from src; + +select distinct src.* from src; + +select distinct * from src; + +explain select distinct * from src where key < '3'; + +select distinct * from src where key < '3'; + +from src a select distinct a.* where a.key = '238'; + +explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq; + +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq; + +drop view if exists sdi; + +explain create view sdi as select distinct * from src order by key limit 2; + +create view sdi as select distinct * from src order by key limit 2; + +describe extended sdi; + +describe formatted sdi; + +select * from sdi; + +select distinct * from src union all select distinct * from src1; + +select distinct * from src join src1 on src.key=src1.key; + +SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key; + +select distinct * from (select distinct * from src)src1; + +set hive.cbo.enable=false; + +explain select distinct src.* from src; + +select distinct src.* from src; + +select distinct * from src; + +explain select distinct * from src where key < '3'; + +select distinct * from src where key < '3'; + +from src a select distinct a.* where a.key = '238'; + +explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq; + +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq; + +drop view if exists sdi; + +explain create view sdi as select distinct * from src order by key limit 2; + +create view sdi as select distinct * from src order by key limit 2; + +describe extended sdi; + +describe formatted sdi; + +select * from sdi; + +select distinct * from src union all select distinct * from src1; + +select distinct * from src join src1 on src.key=src1.key; + +SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key; + +select distinct * from (select distinct * from src)src1; Index: ql/src/test/queries/clientpositive/semijoin.q =================================================================== --- ql/src/test/queries/clientpositive/semijoin.q (revision 1653459) +++ ql/src/test/queries/clientpositive/semijoin.q (working copy) @@ -1,8 +1,5 @@ +-- SORT_QUERY_RESULTS - - - - create table t1 as select cast(key as int) key, value from src where key <= 10; select * from t1 sort by key; Index: ql/src/test/queries/clientpositive/smb_mapjoin_11.q =================================================================== --- ql/src/test/queries/clientpositive/smb_mapjoin_11.q (revision 1653459) +++ ql/src/test/queries/clientpositive/smb_mapjoin_11.q (working copy) @@ -29,6 +29,13 @@ INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1'; +SELECT * FROM test_table1 ORDER BY key; +SELECT * FROM test_table3 ORDER BY key; +EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16); +EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16); +SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16); +SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16); + -- Join data from a sampled bucket to verify the data is bucketed SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1'; Index: ql/src/test/queries/clientpositive/sort.q =================================================================== --- ql/src/test/queries/clientpositive/sort.q (revision 1653459) +++ ql/src/test/queries/clientpositive/sort.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN SELECT x.* FROM SRC x SORT BY key; Index: ql/src/test/queries/clientpositive/stats1.q =================================================================== --- ql/src/test/queries/clientpositive/stats1.q (revision 1653459) +++ ql/src/test/queries/clientpositive/stats1.q (working copy) @@ -4,6 +4,8 @@ set hive.merge.mapredfiles=false; set hive.map.aggr=true; +-- SORT_QUERY_RESULTS + create table tmptable(key string, value string); EXPLAIN @@ -27,4 +29,4 @@ -- Some stats (numFiles, totalSize) should be updated correctly -- Some other stats (numRows, rawDataSize) should be cleared load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE tmptable; -DESCRIBE FORMATTED tmptable; \ No newline at end of file +DESCRIBE FORMATTED tmptable; Index: ql/src/test/queries/clientpositive/transform_ppr1.q =================================================================== --- ql/src/test/queries/clientpositive/transform_ppr1.q (revision 1653459) +++ ql/src/test/queries/clientpositive/transform_ppr1.q (working copy) @@ -1,6 +1,8 @@ set hive.optimize.ppd=true; set hive.entity.capture.transform=true; +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED FROM ( FROM srcpart src Index: ql/src/test/queries/clientpositive/transform_ppr2.q =================================================================== --- ql/src/test/queries/clientpositive/transform_ppr2.q (revision 1653459) +++ ql/src/test/queries/clientpositive/transform_ppr2.q (working copy) @@ -1,6 +1,8 @@ set hive.optimize.ppd=true; set hive.entity.capture.transform=true; +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED FROM ( FROM srcpart src Index: ql/src/test/queries/clientpositive/udf_add_months.q =================================================================== --- ql/src/test/queries/clientpositive/udf_add_months.q (revision 0) +++ ql/src/test/queries/clientpositive/udf_add_months.q (working copy) @@ -0,0 +1,2 @@ +DESCRIBE FUNCTION add_months; +DESCRIBE FUNCTION EXTENDED add_months; Index: ql/src/test/queries/clientpositive/udf_initcap.q =================================================================== --- ql/src/test/queries/clientpositive/udf_initcap.q (revision 0) +++ ql/src/test/queries/clientpositive/udf_initcap.q (working copy) @@ -0,0 +1,2 @@ +DESCRIBE FUNCTION initcap; +DESCRIBE FUNCTION EXTENDED initcap; Index: ql/src/test/queries/clientpositive/udf_last_day.q =================================================================== --- ql/src/test/queries/clientpositive/udf_last_day.q (revision 0) +++ ql/src/test/queries/clientpositive/udf_last_day.q (working copy) @@ -0,0 +1,2 @@ +DESCRIBE FUNCTION last_day; +DESCRIBE FUNCTION EXTENDED last_day; Index: ql/src/test/queries/clientpositive/union10.q =================================================================== --- ql/src/test/queries/clientpositive/union10.q (revision 1653459) +++ ql/src/test/queries/clientpositive/union10.q (working copy) @@ -1,5 +1,7 @@ set hive.map.aggr = true; +-- SORT_QUERY_RESULTS + -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink create table tmptable(key string, value int); Index: ql/src/test/queries/clientpositive/union18.q =================================================================== --- ql/src/test/queries/clientpositive/union18.q (revision 1653459) +++ ql/src/test/queries/clientpositive/union18.q (working copy) @@ -1,6 +1,8 @@ CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE; CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE; +-- SORT_QUERY_RESULTS + -- union case:map-reduce sub-queries followed by multi-table insert explain Index: ql/src/test/queries/clientpositive/union19.q =================================================================== --- ql/src/test/queries/clientpositive/union19.q (revision 1653459) +++ ql/src/test/queries/clientpositive/union19.q (working copy) @@ -1,6 +1,5 @@ +-- SORT_QUERY_RESULTS - - CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE; CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/union6.q =================================================================== --- ql/src/test/queries/clientpositive/union6.q (revision 1653459) +++ ql/src/test/queries/clientpositive/union6.q (working copy) @@ -1,8 +1,9 @@ set hive.map.aggr = true; +-- SORT_QUERY_RESULTS + -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink - create table tmptable(key string, value string); explain Index: ql/src/test/queries/clientpositive/union_ppr.q =================================================================== --- ql/src/test/queries/clientpositive/union_ppr.q (revision 1653459) +++ ql/src/test/queries/clientpositive/union_ppr.q (working copy) @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED SELECT * FROM ( SELECT X.* FROM SRCPART X WHERE X.key < 100 Index: ql/src/test/results/beelinepositive/udf_add_months.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_add_months.q.out (revision 0) +++ ql/src/test/results/beelinepositive/udf_add_months.q.out (working copy) @@ -0,0 +1,15 @@ +Saving all output to "!!{outputDirectory}!!/udf_add_months.q.raw". Enter "record" with no arguments to stop it. +>>> !run !!{qFileDirectory}!!/udf_add_months.q +>>> DESCRIBE FUNCTION add_months; +'tab_name' +'date_add(start_date, num_months) - Returns the date that is num_months after start_date.' +1 row selected +>>> DESCRIBE FUNCTION EXTENDED add_months; +'tab_name' +'date_add(start_date, num_months) - Returns the date that is num_months after start_date.' +'start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_months is a number. The time part of start_date is ignored.' +'Example:' +' > SELECT add_months('2009-08-31', 1) FROM src LIMIT 1;' +' '2009-09-30'' +5 rows selected +>>> !record Index: ql/src/test/results/beelinepositive/udf_date_add.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_date_add.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_date_add.q.out (working copy) @@ -9,7 +9,7 @@ 'date_add(start_date, num_days) - Returns the date that is num_days after start_date.' 'start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_days is a number. The time part of start_date is ignored.' 'Example:' -' > SELECT date_add('2009-30-07', 1) FROM src LIMIT 1;' -' '2009-31-07'' +' > SELECT date_add('2009-07-30', 1) FROM src LIMIT 1;' +' '2009-07-31'' 5 rows selected >>> !record Index: ql/src/test/results/beelinepositive/udf_date_sub.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_date_sub.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_date_sub.q.out (working copy) @@ -9,7 +9,7 @@ 'date_sub(start_date, num_days) - Returns the date that is num_days before start_date.' 'start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_days is a number. The time part of start_date is ignored.' 'Example:' -' > SELECT date_sub('2009-30-07', 1) FROM src LIMIT 1;' -' '2009-29-07'' +' > SELECT date_sub('2009-07-30', 1) FROM src LIMIT 1;' +' '2009-07-29'' 5 rows selected >>> !record Index: ql/src/test/results/beelinepositive/udf_datediff.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_datediff.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_datediff.q.out (working copy) @@ -9,7 +9,7 @@ 'datediff(date1, date2) - Returns the number of days between date1 and date2' 'date1 and date2 are strings in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time parts are ignored.If date1 is earlier than date2, the result is negative.' 'Example:' -' > SELECT datediff('2009-30-07', '2009-31-07') FROM src LIMIT 1;' +' > SELECT datediff('2009-07-30', '2009-07-31') FROM src LIMIT 1;' ' 1' 5 rows selected >>> !record Index: ql/src/test/results/beelinepositive/udf_day.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_day.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_day.q.out (working copy) @@ -10,7 +10,7 @@ 'Synonyms: dayofmonth' 'date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'.' 'Example:' -' > SELECT day('2009-30-07', 1) FROM src LIMIT 1;' +' > SELECT day('2009-07-30') FROM src LIMIT 1;' ' 30' 6 rows selected >>> !record Index: ql/src/test/results/beelinepositive/udf_dayofmonth.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_dayofmonth.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_dayofmonth.q.out (working copy) @@ -10,7 +10,7 @@ 'Synonyms: day' 'date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'.' 'Example:' -' > SELECT dayofmonth('2009-30-07', 1) FROM src LIMIT 1;' +' > SELECT dayofmonth('2009-07-30') FROM src LIMIT 1;' ' 30' 6 rows selected >>> !record Index: ql/src/test/results/beelinepositive/udf_initcap.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_initcap.q.out (revision 0) +++ ql/src/test/results/beelinepositive/udf_initcap.q.out (working copy) @@ -0,0 +1,14 @@ +Saving all output to "!!{outputDirectory}!!/udf_initcap.q.raw". Enter "record" with no arguments to stop it. +>>> !run !!{qFileDirectory}!!/udf_initcap.q +>>> DESCRIBE FUNCTION initcap; +'tab_name' +'initcap(str) - Returns str, with the first letter of each word in uppercase, all other letters in lowercase. Words are delimited by white space.' +1 row selected +>>> DESCRIBE FUNCTION EXTENDED initcap; +'tab_name' +'initcap(str) - Returns str, with the first letter of each word in uppercase, all other letters in lowercase. Words are delimited by white space.' +'Example:' +' > SELECT initcap('tHe soap') FROM src LIMIT 1;' +' 'The Soap'' +5 rows selected +>>> !record Index: ql/src/test/results/beelinepositive/udf_last_day.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_last_day.q.out (revision 0) +++ ql/src/test/results/beelinepositive/udf_last_day.q.out (working copy) @@ -0,0 +1,15 @@ +Saving all output to "!!{outputDirectory}!!/udf_last_day.q.raw". Enter "record" with no arguments to stop it. +>>> !run !!{qFileDirectory}!!/udf_last_day.q +>>> DESCRIBE FUNCTION last_day; +'tab_name' +'last_day(date) - Returns the last day of the month which the date belongs to.' +1 row selected +>>> DESCRIBE FUNCTION EXTENDED last_day; +'tab_name' +'date_add(start_date, num_months) - Returns the last day of the month which the date belongs to.' +'date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_months is a number. The time part of date is ignored.' +'Example:' +' > SELECT last_day('2009-01-12') FROM src LIMIT 1;' +' '2009-01-31'' +5 rows selected +>>> !record Index: ql/src/test/results/beelinepositive/udf_to_date.q.out =================================================================== --- ql/src/test/results/beelinepositive/udf_to_date.q.out (revision 1653459) +++ ql/src/test/results/beelinepositive/udf_to_date.q.out (working copy) @@ -8,7 +8,7 @@ 'tab_name' 'to_date(expr) - Extracts the date part of the date or datetime expression expr' 'Example:' -' > SELECT to_date('2009-30-07 04:17:52') FROM src LIMIT 1;' -' '2009-30-07'' +' > SELECT to_date('2009-07-30 04:17:52') FROM src LIMIT 1;' +' '2009-07-30'' 4 rows selected >>> !record Index: ql/src/test/results/clientnegative/fs_default_name2.q.out =================================================================== --- ql/src/test/results/clientnegative/fs_default_name2.q.out (revision 1653459) +++ ql/src/test/results/clientnegative/fs_default_name2.q.out (working copy) @@ -1 +1 @@ -FAILED: IllegalArgumentException Illegal character in scheme name at index 0: 'http://www.example.com +FAILED: SemanticException java.lang.IllegalArgumentException: Illegal character in scheme name at index 0: 'http://www.example.com Index: ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out =================================================================== --- ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out (revision 0) +++ ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out (working copy) @@ -0,0 +1,9 @@ +PREHOOK: query: -- Duplicate column name: key + +drop view if exists v +PREHOOK: type: DROPVIEW +POSTHOOK: query: -- Duplicate column name: key + +drop view if exists v +POSTHOOK: type: DROPVIEW +FAILED: SemanticException [Error 10036]: Duplicate column name: key Index: ql/src/test/results/clientnegative/selectDistinctStarNeg_2.q.out =================================================================== --- ql/src/test/results/clientnegative/selectDistinctStarNeg_2.q.out (revision 0) +++ ql/src/test/results/clientnegative/selectDistinctStarNeg_2.q.out (working copy) @@ -0,0 +1 @@ +FAILED: SemanticException 3:36 SELECT DISTINCT and GROUP BY can not be in the same query. Error encountered near token 'key' Index: ql/src/test/results/clientpositive/annotate_stats_groupby.q.out =================================================================== --- ql/src/test/results/clientpositive/annotate_stats_groupby.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/annotate_stats_groupby.q.out (working copy) @@ -389,8 +389,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -446,8 +447,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -503,8 +505,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -560,8 +563,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -617,8 +621,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -674,8 +679,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -792,8 +798,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -908,8 +915,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -965,8 +973,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -1022,8 +1031,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -1079,8 +1089,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -1136,8 +1147,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -1193,8 +1205,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 @@ -1307,8 +1320,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 Index: ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out (working copy) @@ -164,8 +164,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -341,8 +342,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -455,8 +457,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Index: ql/src/test/results/clientpositive/auto_join21.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join21.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/auto_join21.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/auto_join23.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join23.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/auto_join23.q.out (working copy) @@ -1,8 +1,12 @@ Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Stage-2:MAPRED' is a cross product -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/cluster.q.out =================================================================== --- ql/src/test/results/clientpositive/cluster.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/cluster.q.out (working copy) @@ -25,7 +25,7 @@ Reduce Output Operator key expressions: '10' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '10' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -83,7 +83,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -141,7 +141,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -199,7 +199,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -257,7 +257,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -315,7 +315,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -430,7 +430,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -485,7 +485,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -497,7 +497,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -576,7 +576,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -588,7 +588,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: @@ -673,7 +673,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -685,7 +685,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: @@ -715,7 +715,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) Reduce Operator Tree: @@ -770,7 +770,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -782,7 +782,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -811,7 +811,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string) Reduce Operator Tree: Index: ql/src/test/results/clientpositive/constprog2.q.out =================================================================== --- ql/src/test/results/clientpositive/constprog2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/constprog2.q.out (working copy) @@ -27,7 +27,7 @@ Reduce Output Operator key expressions: '86' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan @@ -41,7 +41,7 @@ Reduce Output Operator key expressions: '86' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -110,7 +110,7 @@ Reduce Output Operator key expressions: '86' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan @@ -124,7 +124,7 @@ Reduce Output Operator key expressions: '86' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/constprog_partitioner.q.out =================================================================== --- ql/src/test/results/clientpositive/constprog_partitioner.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/constprog_partitioner.q.out (working copy) @@ -27,7 +27,7 @@ Reduce Output Operator key expressions: '100' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '100' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan @@ -41,7 +41,7 @@ Reduce Output Operator key expressions: '100' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '100' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -84,3 +84,102 @@ 100 101.0 val_100 100 101.0 val_100 100 101.0 val_100 +PREHOOK: query: EXPLAIN +SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((l_linenumber = 1) and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col3 (type: int) + Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int) + TableScan + alias: li + Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +PREHOOK: type: QUERY +PREHOOK: Input: default@lineitem +#### A masked pattern was here #### +POSTHOOK: query: SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lineitem +#### A masked pattern was here #### +108570 8571 +4297 1798 Index: ql/src/test/results/clientpositive/correlationoptimizer12.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer12.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/correlationoptimizer12.q.out (working copy) @@ -36,9 +36,11 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -109,9 +111,11 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/ctas_colname.q.out =================================================================== --- ql/src/test/results/clientpositive/ctas_colname.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ctas_colname.q.out (working copy) @@ -183,9 +183,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -329,9 +330,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out (working copy) @@ -0,0 +1,896 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table encryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encryptedTable +POSTHOOK: query: create table encryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encryptedTable +Encryption key created: 'key_1' +Encryption zone created: '/build/ql/test/data/warehouse/encryptedTable' using key: 'key_1' +PREHOOK: query: create table unencryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unencryptedTable +POSTHOOK: query: create table unencryptedTable(value string) + partitioned by (key string) clustered by (value) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unencryptedTable +PREHOOK: query: -- insert encrypted table from values +explain extended insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502') +PREHOOK: type: QUERY +POSTHOOK: query: -- insert encrypted table from values +explain extended insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502') +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + null + null + Values__Tmp__Table__1 + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + encryptedTable + TOK_PARTSPEC + TOK_PARTVAL + key + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + GatherStats: false + Select Operator + expressions: tmp_values_col1 (type: string), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__1 + name: default.values__tmp__table__1 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + NumFilesPerFileSink: 1 + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + + Stage: Stage-2 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + +PREHOOK: query: insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@encryptedtable +POSTHOOK: query: insert into table encryptedTable partition (key) values + ('val_501', '501'), + ('val_502', '502') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@encryptedtable@key=501 +POSTHOOK: Output: default@encryptedtable@key=502 +POSTHOOK: Lineage: encryptedtable PARTITION(key=501).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: encryptedtable PARTITION(key=502).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from encryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@key=501 +PREHOOK: Input: default@encryptedtable@key=502 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +POSTHOOK: query: select * from encryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@key=501 +POSTHOOK: Input: default@encryptedtable@key=502 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +val_501 501 +val_502 502 +PREHOOK: query: -- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition (key) + select value, key limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: -- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition (key) + select value, key limit 2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + encryptedTable + TOK_PARTSPEC + TOK_PARTVAL + key + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_LIMIT + 2 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:src] + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10001 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + + Stage: Stage-3 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging + +PREHOOK: query: from src +insert into table encryptedTable partition (key) + select value, key limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encryptedtable +POSTHOOK: query: from src +insert into table encryptedTable partition (key) + select value, key limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encryptedtable@key=238 +POSTHOOK: Output: default@encryptedtable@key=86 +POSTHOOK: Lineage: encryptedtable PARTITION(key=238).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: encryptedtable PARTITION(key=86).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from encryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@key=238 +PREHOOK: Input: default@encryptedtable@key=501 +PREHOOK: Input: default@encryptedtable@key=502 +PREHOOK: Input: default@encryptedtable@key=86 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +POSTHOOK: query: select * from encryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@key=238 +POSTHOOK: Input: default@encryptedtable@key=501 +POSTHOOK: Input: default@encryptedtable@key=502 +POSTHOOK: Input: default@encryptedtable@key=86 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +val_238 238 +val_501 501 +val_502 502 +val_86 86 +PREHOOK: query: -- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition (key) + select value, key +PREHOOK: type: QUERY +POSTHOOK: query: -- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition (key) + select value, key +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + encryptedTable + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + unencryptedTable + TOK_PARTSPEC + TOK_PARTVAL + key + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + key + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: encryptedtable + Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: key=238 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + key 238 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 1 + numRows 0 + partition_columns key + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 600 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable +#### A masked pattern was here #### + Partition + base file name: key=501 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + key 501 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 1 + numRows 0 + partition_columns key + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 592 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable +#### A masked pattern was here #### + Partition + base file name: key=502 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + key 502 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 1 + numRows 0 + partition_columns key + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 592 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable +#### A masked pattern was here #### + Partition + base file name: key=86 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + key 86 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 1 + numRows 0 + partition_columns key + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 588 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct encryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable + Truncated Path -> Alias: + /encryptedTable/key=238 [encryptedtable] + /encryptedTable/key=501 [encryptedtable] + /encryptedTable/key=502 [encryptedtable] + /encryptedTable/key=86 [encryptedtable] + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/.hive-staging + NumFilesPerFileSink: 1 + Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.unencryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct unencryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.unencryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns value + columns.comments + columns.types string +#### A masked pattern was here #### + name default.unencryptedtable + partition_columns key + partition_columns.types string + serialization.ddl struct unencryptedtable { string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.unencryptedtable + + Stage: Stage-2 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/.hive-staging + +PREHOOK: query: from encryptedTable +insert into table unencryptedTable partition (key) + select value, key +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@key=238 +PREHOOK: Input: default@encryptedtable@key=501 +PREHOOK: Input: default@encryptedtable@key=502 +PREHOOK: Input: default@encryptedtable@key=86 +PREHOOK: Output: default@unencryptedtable +POSTHOOK: query: from encryptedTable +insert into table unencryptedTable partition (key) + select value, key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@key=238 +POSTHOOK: Input: default@encryptedtable@key=501 +POSTHOOK: Input: default@encryptedtable@key=502 +POSTHOOK: Input: default@encryptedtable@key=86 +POSTHOOK: Output: default@unencryptedtable@key=238 +POSTHOOK: Output: default@unencryptedtable@key=501 +POSTHOOK: Output: default@unencryptedtable@key=502 +POSTHOOK: Output: default@unencryptedtable@key=86 +POSTHOOK: Lineage: unencryptedtable PARTITION(key=238).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: unencryptedtable PARTITION(key=501).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: unencryptedtable PARTITION(key=502).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: unencryptedtable PARTITION(key=86).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from unencryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@unencryptedtable +PREHOOK: Input: default@unencryptedtable@key=238 +PREHOOK: Input: default@unencryptedtable@key=501 +PREHOOK: Input: default@unencryptedtable@key=502 +PREHOOK: Input: default@unencryptedtable@key=86 +#### A masked pattern was here #### +POSTHOOK: query: select * from unencryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unencryptedtable +POSTHOOK: Input: default@unencryptedtable@key=238 +POSTHOOK: Input: default@unencryptedtable@key=501 +POSTHOOK: Input: default@unencryptedtable@key=502 +POSTHOOK: Input: default@unencryptedtable@key=86 +#### A masked pattern was here #### +val_238 238 +val_501 501 +val_502 502 +val_86 86 +PREHOOK: query: -- clean up +drop table encryptedTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encryptedtable +PREHOOK: Output: default@encryptedtable +POSTHOOK: query: -- clean up +drop table encryptedTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Output: default@encryptedtable +Encryption key deleted: 'key_1' +PREHOOK: query: drop table unencryptedTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unencryptedtable +PREHOOK: Output: default@unencryptedtable +POSTHOOK: query: drop table unencryptedTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unencryptedtable +POSTHOOK: Output: default@unencryptedtable Index: ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out (working copy) @@ -0,0 +1,789 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- init +drop table IF EXISTS encryptedTable +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table encryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encryptedTable +POSTHOOK: query: create table encryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encryptedTable +Encryption key created: 'key_1' +Encryption zone created: '/build/ql/test/data/warehouse/encryptedTable' using key: 'key_1' +PREHOOK: query: create table unencryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unencryptedTable +POSTHOOK: query: create table unencryptedTable(key string, + value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unencryptedTable +PREHOOK: query: -- insert encrypted table from values +explain extended insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502') +PREHOOK: type: QUERY +POSTHOOK: query: -- insert encrypted table from values +explain extended insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502') +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + null + null + Values__Tmp__Table__1 + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + encryptedTable + TOK_PARTSPEC + TOK_PARTVAL + ds + 'today' + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + GatherStats: false + Select Operator + expressions: tmp_values_col1 (type: string), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__1 + name: default.values__tmp__table__1 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging + NumFilesPerFileSink: 1 + Static Partition Specification: ds=today/ + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + ds today + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + + Stage: Stage-2 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging + +PREHOOK: query: insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@encryptedtable@ds=today +POSTHOOK: query: insert into table encryptedTable partition + (ds='today') values + ('501', 'val_501'), + ('502', 'val_502') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@encryptedtable@ds=today +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select * from encryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@ds=today +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +POSTHOOK: query: select * from encryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@ds=today +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +501 val_501 today +502 val_502 today +PREHOOK: query: -- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: -- insert encrypted table from unencrypted source +explain extended from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + encryptedTable + TOK_PARTSPEC + TOK_PARTVAL + ds + 'yesterday' + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_LIMIT + 2 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10001 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging + NumFilesPerFileSink: 1 + Static Partition Specification: ds=yesterday/ + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + ds yesterday + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + + Stage: Stage-3 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging + +PREHOOK: query: from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encryptedtable@ds=yesterday +POSTHOOK: query: from src +insert into table encryptedTable partition + (ds='yesterday') + select * limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encryptedtable@ds=yesterday +POSTHOOK: Lineage: encryptedtable PARTITION(ds=yesterday).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encryptedtable PARTITION(ds=yesterday).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from encryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@ds=today +PREHOOK: Input: default@encryptedtable@ds=yesterday +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +POSTHOOK: query: select * from encryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@ds=today +POSTHOOK: Input: default@encryptedtable@ds=yesterday +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging +238 val_238 yesterday +501 val_501 today +502 val_502 today +86 val_86 yesterday +PREHOOK: query: -- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value +PREHOOK: type: QUERY +POSTHOOK: query: -- insert unencrypted table from encrypted source +explain extended from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + encryptedTable + TOK_INSERT + TOK_INSERT_INTO + TOK_TAB + TOK_TABNAME + unencryptedTable + TOK_PARTSPEC + TOK_PARTVAL + ds + 'today' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: encryptedtable + Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=today + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + ds today + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 1351 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable +#### A masked pattern was here #### + Partition + base file name: ds=yesterday + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + ds yesterday + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 1344 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.encryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct encryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.encryptedtable + name: default.encryptedtable + Truncated Path -> Alias: + /encryptedTable/ds=today [encryptedtable] + /encryptedTable/ds=yesterday [encryptedtable] + Needs Tagging: false + Reduce Operator Tree: + Extract + Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging + NumFilesPerFileSink: 1 + Static Partition Specification: ds=today/ + Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.unencryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct unencryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.unencryptedtable + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + ds today + replace: false +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.unencryptedtable + partition_columns ds + partition_columns.types string + serialization.ddl struct unencryptedtable { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + transactional true +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.unencryptedtable + + Stage: Stage-2 + Stats-Aggr Operator +#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging + +PREHOOK: query: from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@encryptedtable +PREHOOK: Input: default@encryptedtable@ds=today +PREHOOK: Input: default@encryptedtable@ds=yesterday +PREHOOK: Output: default@unencryptedtable@ds=today +POSTHOOK: query: from encryptedTable +insert into table unencryptedTable partition + (ds='today') + select key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Input: default@encryptedtable@ds=today +POSTHOOK: Input: default@encryptedtable@ds=yesterday +POSTHOOK: Output: default@unencryptedtable@ds=today +POSTHOOK: Lineage: unencryptedtable PARTITION(ds=today).key SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: unencryptedtable PARTITION(ds=today).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from unencryptedTable order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@unencryptedtable +PREHOOK: Input: default@unencryptedtable@ds=today +#### A masked pattern was here #### +POSTHOOK: query: select * from unencryptedTable order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unencryptedtable +POSTHOOK: Input: default@unencryptedtable@ds=today +#### A masked pattern was here #### +238 val_238 today +501 val_501 today +502 val_502 today +86 val_86 today +PREHOOK: query: -- clean up +drop table encryptedTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encryptedtable +PREHOOK: Output: default@encryptedtable +POSTHOOK: query: -- clean up +drop table encryptedTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encryptedtable +POSTHOOK: Output: default@encryptedtable +Encryption key deleted: 'key_1' +PREHOOK: query: drop table unencryptedTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unencryptedtable +PREHOOK: Output: default@unencryptedtable +POSTHOOK: query: drop table unencryptedTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unencryptedtable +POSTHOOK: Output: default@unencryptedtable Index: ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out (working copy) @@ -0,0 +1,763 @@ +PREHOOK: query: --SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: --SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + t1 + TOK_TABREF + TOK_TABNAME + encrypted_table + t2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + = + . + TOK_TABLE_OR_COL + t1 + key + . + TOK_TABLE_OR_COL + t2 + key + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: key (type: string), value (type: string) + auto parallelism: false + TableScan + alias: t2 + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: int), value (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: encrypted_table + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.encrypted_table + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct encrypted_table { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.encrypted_table + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct encrypted_table { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.encrypted_table + name: default.encrypted_table +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /default/encrypted_table [t2] + /src [t1] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: (_col0 = _col5) (type: boolean) + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging + NumFilesPerFileSink: 1 + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: drop table encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +Encryption key deleted: 'key_128' Index: ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out (working copy) @@ -0,0 +1,1330 @@ +PREHOOK: query: --SORT_QUERY_RESULTS + +-- Java JCE must be installed in order to hava a key length of 256 bits +DROP TABLE IF EXISTS table_key_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: --SORT_QUERY_RESULTS + +-- Java JCE must be installed in order to hava a key length of 256 bits +DROP TABLE IF EXISTS table_key_1 +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@table_key_1 +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_key_1 +Encryption key created: 'key_1' +Encryption zone created: '/build/ql/test/data/warehouse/table_key_1' using key: 'key_1' +PREHOOK: query: DROP TABLE IF EXISTS table_key_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@table_key_2 +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_key_2 +Encryption key created: 'key_2' +Encryption zone created: '/build/ql/test/data/warehouse/table_key_2' using key: 'key_2' +PREHOOK: query: INSERT OVERWRITE TABLE table_key_1 SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@table_key_1 +POSTHOOK: query: INSERT OVERWRITE TABLE table_key_1 SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@table_key_1 +POSTHOOK: Lineage: table_key_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table_key_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@table_key_2 +POSTHOOK: query: INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@table_key_2 +POSTHOOK: Lineage: table_key_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table_key_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + table_key_1 + t1 + TOK_TABREF + TOK_TABNAME + table_key_2 + t2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + = + . + TOK_TABLE_OR_COL + t1 + key + . + TOK_TABLE_OR_COL + t2 + key + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: false + TableScan + alias: t2 + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: table_key_1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.table_key_1 + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct table_key_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.table_key_1 + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct table_key_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table_key_1 + name: default.table_key_1 +#### A masked pattern was here #### + Partition + base file name: table_key_2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.table_key_2 + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct table_key_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.table_key_2 + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct table_key_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table_key_2 + name: default.table_key_2 + Truncated Path -> Alias: + /table_key_1 [t1] + /table_key_2 [t2] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: (_col0 = _col5) (type: boolean) + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A PARTIAL masked pattern was here #### data/warehouse/table_key_2/.hive-staging + NumFilesPerFileSink: 1 + Statistics: Num rows: 15 Data size: 1626 Basic stats: COMPLETE Column stats: NONE +#### A PARTIAL masked pattern was here #### data/warehouse/table_key_2/.hive-staging + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@table_key_1 +PREHOOK: Input: default@table_key_2 +#### A PARTIAL masked pattern was here #### data/warehouse/table_key_2/.hive-staging +POSTHOOK: query: SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@table_key_1 +POSTHOOK: Input: default@table_key_2 +#### A PARTIAL masked pattern was here #### data/warehouse/table_key_2/.hive-staging +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +10 val_10 10 val_10 +11 val_11 11 val_11 +12 val_12 12 val_12 +12 val_12 12 val_12 +12 val_12 12 val_12 +12 val_12 12 val_12 +15 val_15 15 val_15 +15 val_15 15 val_15 +15 val_15 15 val_15 +15 val_15 15 val_15 +17 val_17 17 val_17 +18 val_18 18 val_18 +18 val_18 18 val_18 +18 val_18 18 val_18 +18 val_18 18 val_18 +19 val_19 19 val_19 +20 val_20 20 val_20 +24 val_24 24 val_24 +24 val_24 24 val_24 +24 val_24 24 val_24 +24 val_24 24 val_24 +26 val_26 26 val_26 +26 val_26 26 val_26 +26 val_26 26 val_26 +26 val_26 26 val_26 +27 val_27 27 val_27 +28 val_28 28 val_28 +30 val_30 30 val_30 +33 val_33 33 val_33 +34 val_34 34 val_34 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +37 val_37 37 val_37 +37 val_37 37 val_37 +37 val_37 37 val_37 +37 val_37 37 val_37 +41 val_41 41 val_41 +42 val_42 42 val_42 +42 val_42 42 val_42 +42 val_42 42 val_42 +42 val_42 42 val_42 +43 val_43 43 val_43 +44 val_44 44 val_44 +47 val_47 47 val_47 +51 val_51 51 val_51 +51 val_51 51 val_51 +51 val_51 51 val_51 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +58 val_58 58 val_58 +58 val_58 58 val_58 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +67 val_67 67 val_67 +67 val_67 67 val_67 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +72 val_72 72 val_72 +72 val_72 72 val_72 +72 val_72 72 val_72 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +76 val_76 76 val_76 +76 val_76 76 val_76 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +83 val_83 83 val_83 +83 val_83 83 val_83 +83 val_83 83 val_83 +84 val_84 84 val_84 +84 val_84 84 val_84 +84 val_84 84 val_84 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +95 val_95 95 val_95 +95 val_95 95 val_95 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +97 val_97 97 val_97 +97 val_97 97 val_97 +97 val_97 97 val_97 +98 val_98 98 val_98 +98 val_98 98 val_98 +98 val_98 98 val_98 +98 val_98 98 val_98 +100 val_100 100 val_100 +100 val_100 100 val_100 +100 val_100 100 val_100 +100 val_100 100 val_100 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 +104 val_104 104 val_104 +104 val_104 104 val_104 +104 val_104 104 val_104 +104 val_104 104 val_104 +105 val_105 105 val_105 +111 val_111 111 val_111 +113 val_113 113 val_113 +113 val_113 113 val_113 +113 val_113 113 val_113 +113 val_113 113 val_113 +114 val_114 114 val_114 +116 val_116 116 val_116 +118 val_118 118 val_118 +118 val_118 118 val_118 +118 val_118 118 val_118 +118 val_118 118 val_118 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +119 val_119 119 val_119 +120 val_120 120 val_120 +120 val_120 120 val_120 +120 val_120 120 val_120 +120 val_120 120 val_120 +125 val_125 125 val_125 +125 val_125 125 val_125 +125 val_125 125 val_125 +125 val_125 125 val_125 +126 val_126 126 val_126 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +128 val_128 128 val_128 +129 val_129 129 val_129 +129 val_129 129 val_129 +129 val_129 129 val_129 +129 val_129 129 val_129 +131 val_131 131 val_131 +133 val_133 133 val_133 +134 val_134 134 val_134 +134 val_134 134 val_134 +134 val_134 134 val_134 +134 val_134 134 val_134 +136 val_136 136 val_136 +137 val_137 137 val_137 +137 val_137 137 val_137 +137 val_137 137 val_137 +137 val_137 137 val_137 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +138 val_138 138 val_138 +143 val_143 143 val_143 +145 val_145 145 val_145 +146 val_146 146 val_146 +146 val_146 146 val_146 +146 val_146 146 val_146 +146 val_146 146 val_146 +149 val_149 149 val_149 +149 val_149 149 val_149 +149 val_149 149 val_149 +149 val_149 149 val_149 +150 val_150 150 val_150 +152 val_152 152 val_152 +152 val_152 152 val_152 +152 val_152 152 val_152 +152 val_152 152 val_152 +153 val_153 153 val_153 +155 val_155 155 val_155 +156 val_156 156 val_156 +157 val_157 157 val_157 +158 val_158 158 val_158 +160 val_160 160 val_160 +162 val_162 162 val_162 +163 val_163 163 val_163 +164 val_164 164 val_164 +164 val_164 164 val_164 +164 val_164 164 val_164 +164 val_164 164 val_164 +165 val_165 165 val_165 +165 val_165 165 val_165 +165 val_165 165 val_165 +165 val_165 165 val_165 +166 val_166 166 val_166 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +167 val_167 167 val_167 +168 val_168 168 val_168 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +169 val_169 169 val_169 +170 val_170 170 val_170 +172 val_172 172 val_172 +172 val_172 172 val_172 +172 val_172 172 val_172 +172 val_172 172 val_172 +174 val_174 174 val_174 +174 val_174 174 val_174 +174 val_174 174 val_174 +174 val_174 174 val_174 +175 val_175 175 val_175 +175 val_175 175 val_175 +175 val_175 175 val_175 +175 val_175 175 val_175 +176 val_176 176 val_176 +176 val_176 176 val_176 +176 val_176 176 val_176 +176 val_176 176 val_176 +177 val_177 177 val_177 +178 val_178 178 val_178 +179 val_179 179 val_179 +179 val_179 179 val_179 +179 val_179 179 val_179 +179 val_179 179 val_179 +180 val_180 180 val_180 +181 val_181 181 val_181 +183 val_183 183 val_183 +186 val_186 186 val_186 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +187 val_187 187 val_187 +189 val_189 189 val_189 +190 val_190 190 val_190 +191 val_191 191 val_191 +191 val_191 191 val_191 +191 val_191 191 val_191 +191 val_191 191 val_191 +192 val_192 192 val_192 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +193 val_193 193 val_193 +194 val_194 194 val_194 +195 val_195 195 val_195 +195 val_195 195 val_195 +195 val_195 195 val_195 +195 val_195 195 val_195 +196 val_196 196 val_196 +197 val_197 197 val_197 +197 val_197 197 val_197 +197 val_197 197 val_197 +197 val_197 197 val_197 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +199 val_199 199 val_199 +200 val_200 200 val_200 +200 val_200 200 val_200 +200 val_200 200 val_200 +200 val_200 200 val_200 +201 val_201 201 val_201 +202 val_202 202 val_202 +203 val_203 203 val_203 +203 val_203 203 val_203 +203 val_203 203 val_203 +203 val_203 203 val_203 +205 val_205 205 val_205 +205 val_205 205 val_205 +205 val_205 205 val_205 +205 val_205 205 val_205 +207 val_207 207 val_207 +207 val_207 207 val_207 +207 val_207 207 val_207 +207 val_207 207 val_207 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +208 val_208 208 val_208 +209 val_209 209 val_209 +209 val_209 209 val_209 +209 val_209 209 val_209 +209 val_209 209 val_209 +213 val_213 213 val_213 +213 val_213 213 val_213 +213 val_213 213 val_213 +213 val_213 213 val_213 +214 val_214 214 val_214 +216 val_216 216 val_216 +216 val_216 216 val_216 +216 val_216 216 val_216 +216 val_216 216 val_216 +217 val_217 217 val_217 +217 val_217 217 val_217 +217 val_217 217 val_217 +217 val_217 217 val_217 +218 val_218 218 val_218 +219 val_219 219 val_219 +219 val_219 219 val_219 +219 val_219 219 val_219 +219 val_219 219 val_219 +221 val_221 221 val_221 +221 val_221 221 val_221 +221 val_221 221 val_221 +221 val_221 221 val_221 +222 val_222 222 val_222 +223 val_223 223 val_223 +223 val_223 223 val_223 +223 val_223 223 val_223 +223 val_223 223 val_223 +224 val_224 224 val_224 +224 val_224 224 val_224 +224 val_224 224 val_224 +224 val_224 224 val_224 +226 val_226 226 val_226 +228 val_228 228 val_228 +229 val_229 229 val_229 +229 val_229 229 val_229 +229 val_229 229 val_229 +229 val_229 229 val_229 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +230 val_230 230 val_230 +233 val_233 233 val_233 +233 val_233 233 val_233 +233 val_233 233 val_233 +233 val_233 233 val_233 +235 val_235 235 val_235 +237 val_237 237 val_237 +237 val_237 237 val_237 +237 val_237 237 val_237 +237 val_237 237 val_237 +238 val_238 238 val_238 +238 val_238 238 val_238 +238 val_238 238 val_238 +238 val_238 238 val_238 +239 val_239 239 val_239 +239 val_239 239 val_239 +239 val_239 239 val_239 +239 val_239 239 val_239 +241 val_241 241 val_241 +242 val_242 242 val_242 +242 val_242 242 val_242 +242 val_242 242 val_242 +242 val_242 242 val_242 +244 val_244 244 val_244 +247 val_247 247 val_247 +248 val_248 248 val_248 +249 val_249 249 val_249 +252 val_252 252 val_252 +255 val_255 255 val_255 +255 val_255 255 val_255 +255 val_255 255 val_255 +255 val_255 255 val_255 +256 val_256 256 val_256 +256 val_256 256 val_256 +256 val_256 256 val_256 +256 val_256 256 val_256 +257 val_257 257 val_257 +258 val_258 258 val_258 +260 val_260 260 val_260 +262 val_262 262 val_262 +263 val_263 263 val_263 +265 val_265 265 val_265 +265 val_265 265 val_265 +265 val_265 265 val_265 +265 val_265 265 val_265 +266 val_266 266 val_266 +272 val_272 272 val_272 +272 val_272 272 val_272 +272 val_272 272 val_272 +272 val_272 272 val_272 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +273 val_273 273 val_273 +274 val_274 274 val_274 +275 val_275 275 val_275 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +277 val_277 277 val_277 +278 val_278 278 val_278 +278 val_278 278 val_278 +278 val_278 278 val_278 +278 val_278 278 val_278 +280 val_280 280 val_280 +280 val_280 280 val_280 +280 val_280 280 val_280 +280 val_280 280 val_280 +281 val_281 281 val_281 +281 val_281 281 val_281 +281 val_281 281 val_281 +281 val_281 281 val_281 +282 val_282 282 val_282 +282 val_282 282 val_282 +282 val_282 282 val_282 +282 val_282 282 val_282 +283 val_283 283 val_283 +284 val_284 284 val_284 +285 val_285 285 val_285 +286 val_286 286 val_286 +287 val_287 287 val_287 +288 val_288 288 val_288 +288 val_288 288 val_288 +288 val_288 288 val_288 +288 val_288 288 val_288 +289 val_289 289 val_289 +291 val_291 291 val_291 +292 val_292 292 val_292 +296 val_296 296 val_296 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +298 val_298 298 val_298 +302 val_302 302 val_302 +305 val_305 305 val_305 +306 val_306 306 val_306 +307 val_307 307 val_307 +307 val_307 307 val_307 +307 val_307 307 val_307 +307 val_307 307 val_307 +308 val_308 308 val_308 +309 val_309 309 val_309 +309 val_309 309 val_309 +309 val_309 309 val_309 +309 val_309 309 val_309 +310 val_310 310 val_310 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +311 val_311 311 val_311 +315 val_315 315 val_315 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +316 val_316 316 val_316 +317 val_317 317 val_317 +317 val_317 317 val_317 +317 val_317 317 val_317 +317 val_317 317 val_317 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +318 val_318 318 val_318 +321 val_321 321 val_321 +321 val_321 321 val_321 +321 val_321 321 val_321 +321 val_321 321 val_321 +322 val_322 322 val_322 +322 val_322 322 val_322 +322 val_322 322 val_322 +322 val_322 322 val_322 +323 val_323 323 val_323 +325 val_325 325 val_325 +325 val_325 325 val_325 +325 val_325 325 val_325 +325 val_325 325 val_325 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +327 val_327 327 val_327 +331 val_331 331 val_331 +331 val_331 331 val_331 +331 val_331 331 val_331 +331 val_331 331 val_331 +332 val_332 332 val_332 +333 val_333 333 val_333 +333 val_333 333 val_333 +333 val_333 333 val_333 +333 val_333 333 val_333 +335 val_335 335 val_335 +336 val_336 336 val_336 +338 val_338 338 val_338 +339 val_339 339 val_339 +341 val_341 341 val_341 +342 val_342 342 val_342 +342 val_342 342 val_342 +342 val_342 342 val_342 +342 val_342 342 val_342 +344 val_344 344 val_344 +344 val_344 344 val_344 +344 val_344 344 val_344 +344 val_344 344 val_344 +345 val_345 345 val_345 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +348 val_348 348 val_348 +351 val_351 351 val_351 +353 val_353 353 val_353 +353 val_353 353 val_353 +353 val_353 353 val_353 +353 val_353 353 val_353 +356 val_356 356 val_356 +360 val_360 360 val_360 +362 val_362 362 val_362 +364 val_364 364 val_364 +365 val_365 365 val_365 +366 val_366 366 val_366 +367 val_367 367 val_367 +367 val_367 367 val_367 +367 val_367 367 val_367 +367 val_367 367 val_367 +368 val_368 368 val_368 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +369 val_369 369 val_369 +373 val_373 373 val_373 +374 val_374 374 val_374 +375 val_375 375 val_375 +377 val_377 377 val_377 +378 val_378 378 val_378 +379 val_379 379 val_379 +382 val_382 382 val_382 +382 val_382 382 val_382 +382 val_382 382 val_382 +382 val_382 382 val_382 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +384 val_384 384 val_384 +386 val_386 386 val_386 +389 val_389 389 val_389 +392 val_392 392 val_392 +393 val_393 393 val_393 +394 val_394 394 val_394 +395 val_395 395 val_395 +395 val_395 395 val_395 +395 val_395 395 val_395 +395 val_395 395 val_395 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +396 val_396 396 val_396 +397 val_397 397 val_397 +397 val_397 397 val_397 +397 val_397 397 val_397 +397 val_397 397 val_397 +399 val_399 399 val_399 +399 val_399 399 val_399 +399 val_399 399 val_399 +399 val_399 399 val_399 +400 val_400 400 val_400 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +401 val_401 401 val_401 +402 val_402 402 val_402 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +403 val_403 403 val_403 +404 val_404 404 val_404 +404 val_404 404 val_404 +404 val_404 404 val_404 +404 val_404 404 val_404 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +406 val_406 406 val_406 +407 val_407 407 val_407 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +409 val_409 409 val_409 +411 val_411 411 val_411 +413 val_413 413 val_413 +413 val_413 413 val_413 +413 val_413 413 val_413 +413 val_413 413 val_413 +414 val_414 414 val_414 +414 val_414 414 val_414 +414 val_414 414 val_414 +414 val_414 414 val_414 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +417 val_417 417 val_417 +418 val_418 418 val_418 +419 val_419 419 val_419 +421 val_421 421 val_421 +424 val_424 424 val_424 +424 val_424 424 val_424 +424 val_424 424 val_424 +424 val_424 424 val_424 +427 val_427 427 val_427 +429 val_429 429 val_429 +429 val_429 429 val_429 +429 val_429 429 val_429 +429 val_429 429 val_429 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +430 val_430 430 val_430 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +431 val_431 431 val_431 +432 val_432 432 val_432 +435 val_435 435 val_435 +436 val_436 436 val_436 +437 val_437 437 val_437 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +438 val_438 438 val_438 +439 val_439 439 val_439 +439 val_439 439 val_439 +439 val_439 439 val_439 +439 val_439 439 val_439 +443 val_443 443 val_443 +444 val_444 444 val_444 +446 val_446 446 val_446 +448 val_448 448 val_448 +449 val_449 449 val_449 +452 val_452 452 val_452 +453 val_453 453 val_453 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +454 val_454 454 val_454 +455 val_455 455 val_455 +457 val_457 457 val_457 +458 val_458 458 val_458 +458 val_458 458 val_458 +458 val_458 458 val_458 +458 val_458 458 val_458 +459 val_459 459 val_459 +459 val_459 459 val_459 +459 val_459 459 val_459 +459 val_459 459 val_459 +460 val_460 460 val_460 +462 val_462 462 val_462 +462 val_462 462 val_462 +462 val_462 462 val_462 +462 val_462 462 val_462 +463 val_463 463 val_463 +463 val_463 463 val_463 +463 val_463 463 val_463 +463 val_463 463 val_463 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +466 val_466 466 val_466 +467 val_467 467 val_467 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +468 val_468 468 val_468 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +469 val_469 469 val_469 +470 val_470 470 val_470 +472 val_472 472 val_472 +475 val_475 475 val_475 +477 val_477 477 val_477 +478 val_478 478 val_478 +478 val_478 478 val_478 +478 val_478 478 val_478 +478 val_478 478 val_478 +479 val_479 479 val_479 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +480 val_480 480 val_480 +481 val_481 481 val_481 +482 val_482 482 val_482 +483 val_483 483 val_483 +484 val_484 484 val_484 +485 val_485 485 val_485 +487 val_487 487 val_487 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +489 val_489 489 val_489 +490 val_490 490 val_490 +491 val_491 491 val_491 +492 val_492 492 val_492 +492 val_492 492 val_492 +492 val_492 492 val_492 +492 val_492 492 val_492 +493 val_493 493 val_493 +494 val_494 494 val_494 +495 val_495 495 val_495 +496 val_496 496 val_496 +497 val_497 497 val_497 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +498 val_498 498 val_498 +PREHOOK: query: DROP TABLE table_key_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table_key_1 +PREHOOK: Output: default@table_key_1 +POSTHOOK: query: DROP TABLE table_key_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table_key_1 +POSTHOOK: Output: default@table_key_1 +PREHOOK: query: DROP TABLE table_key_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table_key_2 +PREHOOK: Output: default@table_key_2 +POSTHOOK: query: DROP TABLE table_key_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table_key_2 +POSTHOOK: Output: default@table_key_2 +Encryption key deleted: 'key_1' +Encryption key deleted: 'key_2' Index: ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out (working copy) @@ -0,0 +1,1059 @@ +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key1' +Encryption zone created: '/build/ql/test/data/warehouse/encrypted_table' using key: 'key1' +PREHOOK: query: -- Test loading data from the local filesystem; +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE encrypted_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: -- Test loading data from the local filesystem; +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE encrypted_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SELECT * FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 +#### A masked pattern was here #### +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SELECT * FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 +PREHOOK: query: DROP TABLE encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +Encryption key deleted: 'key1' Index: ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out (working copy) @@ -0,0 +1,45 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS encrypted_table +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE encrypted_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE encrypted_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SELECT count(*) FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A masked pattern was here #### +500 +PREHOOK: query: drop table encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: drop table encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table Index: ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_unencrypted_tbl.q.out =================================================================== --- ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_unencrypted_tbl.q.out (revision 0) +++ ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_unencrypted_tbl.q.out (working copy) @@ -0,0 +1,43 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS unencrypted_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS unencrypted_table +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@unencrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unencrypted_table +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE unencrypted_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@unencrypted_table +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE unencrypted_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@unencrypted_table +PREHOOK: query: SELECT count(*) FROM unencrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@unencrypted_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM unencrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unencrypted_table +#### A masked pattern was here #### +500 +PREHOOK: query: drop table unencrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unencrypted_table +PREHOOK: Output: default@unencrypted_table +POSTHOOK: query: drop table unencrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unencrypted_table +POSTHOOK: Output: default@unencrypted_table Index: ql/src/test/results/clientpositive/groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby2.q.out (working copy) @@ -81,11 +81,15 @@ POSTHOOK: Lineage: dest_g2.c1 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.c2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT dest_g2.* FROM dest_g2 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 PREHOOK: type: QUERY PREHOOK: Input: default@dest_g2 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest_g2.* FROM dest_g2 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_g2 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/groupby_cube1.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_cube1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_cube1.q.out (working copy) @@ -56,8 +56,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -223,8 +224,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -320,8 +322,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -403,8 +406,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: complete - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -541,8 +545,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 @@ -608,8 +613,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out (working copy) @@ -79,8 +79,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -185,8 +186,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: double) outputColumnNames: _col0, _col1, _col2 @@ -313,8 +315,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 12 Data size: 84 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out (working copy) @@ -74,8 +74,9 @@ aggregations: avg(VALUE._col0), count(VALUE._col1) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: double), _col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 @@ -185,8 +186,9 @@ aggregations: avg(VALUE._col0), count(VALUE._col1) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: double), _col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Index: ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out (working copy) @@ -71,8 +71,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -148,8 +149,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -273,8 +275,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -374,8 +377,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out (working copy) @@ -88,8 +88,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -228,8 +229,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out (working copy) @@ -55,8 +55,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 @@ -140,8 +141,9 @@ Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Index: ql/src/test/results/clientpositive/groupby_grouping_window.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_grouping_window.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby_grouping_window.q.out (working copy) @@ -0,0 +1,134 @@ +PREHOOK: query: create table t(category int, live int, comments int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t(category int, live int, comments int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into table t select key, 0, 2 from src tablesample(3 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t +POSTHOOK: query: insert into table t select key, 0, 2 from src tablesample(3 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.category EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t.comments SIMPLE [] +POSTHOOK: Lineage: t.live SIMPLE [] +PREHOOK: query: explain +select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: category (type: int), live (type: int), comments (type: int) + outputColumnNames: category, live, comments + Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(live), max(comments) + keys: category (type: int), '0' (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int), _col3 (type: int) + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), max(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true + Filter Operator + predicate: (_col3 > 0) (type: boolean) + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _wcol0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select category, max(live) live, max(comments) comments, rank() OVER (PARTITION BY category ORDER BY comments) rank1 +FROM t +GROUP BY category +GROUPING SETS ((), (category)) +HAVING max(comments) > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +NULL 0 2 1 +86 0 2 1 +238 0 2 1 +311 0 2 1 Index: ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out (working copy) @@ -14,12 +14,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest_g3 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT src.key), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1) PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT src.key), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1) Index: ql/src/test/results/clientpositive/groupby_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_ppr.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_ppr.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 Index: ql/src/test/results/clientpositive/groupby_resolution.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_resolution.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_resolution.q.out (working copy) @@ -704,7 +704,9 @@ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: bigint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/groupby_rollup1.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby_rollup1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/groupby_rollup1.q.out (working copy) @@ -56,8 +56,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -133,8 +134,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -230,8 +232,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -307,8 +310,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: complete - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -445,8 +449,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 @@ -512,8 +517,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/identity_project_remove_skip.q.out =================================================================== --- ql/src/test/results/clientpositive/identity_project_remove_skip.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/identity_project_remove_skip.q.out (working copy) @@ -135,13 +135,13 @@ Reduce Output Operator key expressions: '105' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '105' (type: string) Statistics: Num rows: 62 Data size: 658 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: '105' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '105' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out =================================================================== --- ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out (working copy) @@ -63,8 +63,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -143,9 +144,9 @@ InputFormat: org.apache.hadoop.mapred.TextInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] +Num Buckets: 1 +Bucket Columns: [key, value] +Sort Columns: [Order(col:key, order:1), Order(col:value, order:1)] Storage Desc Params: serialization.format 1 PREHOOK: query: -- Test rollup, should be bucketed and sorted on key, value, grouping_key @@ -250,8 +251,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -330,9 +332,9 @@ InputFormat: org.apache.hadoop.mapred.TextInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] +Num Buckets: 1 +Bucket Columns: [key, value] +Sort Columns: [Order(col:key, order:1), Order(col:value, order:1)] Storage Desc Params: serialization.format 1 PREHOOK: query: -- Test cube, should be bucketed and sorted on key, value, grouping_key @@ -437,8 +439,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -517,9 +520,9 @@ InputFormat: org.apache.hadoop.mapred.TextInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] +Num Buckets: 1 +Bucket Columns: [key, value] +Sort Columns: [Order(col:key, order:1), Order(col:value, order:1)] Storage Desc Params: serialization.format 1 PREHOOK: query: -- Test grouping sets, should be bucketed and sorted on key, value, grouping_key Index: ql/src/test/results/clientpositive/input14.q.out =================================================================== --- ql/src/test/results/clientpositive/input14.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/input14.q.out (working copy) @@ -105,11 +105,15 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/input17.q.out =================================================================== --- ql/src/test/results/clientpositive/input17.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/input17.q.out (working copy) @@ -102,15 +102,18 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### -NULL NULL -1461153966 {"myint":49,"mystring":"343","underscore_int":7} -1952710705 {"myint":25,"mystring":"125","underscore_int":5} -734328905 {"myint":16,"mystring":"64","underscore_int":4} @@ -121,3 +124,4 @@ 336964422 {"myint":81,"mystring":"729","underscore_int":9} 465985201 {"myint":1,"mystring":"1","underscore_int":1} 477111225 {"myint":9,"mystring":"27","underscore_int":3} +NULL NULL Index: ql/src/test/results/clientpositive/input18.q.out =================================================================== --- ql/src/test/results/clientpositive/input18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/input18.q.out (working copy) @@ -105,11 +105,15 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/input_part2.q.out =================================================================== --- ql/src/test/results/clientpositive/input_part2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/input_part2.q.out (working copy) @@ -14,12 +14,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest2 -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' INSERT OVERWRITE TABLE dest2 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-09' and srcpart.hr = '12' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' INSERT OVERWRITE TABLE dest2 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-09' and srcpart.hr = '12' @@ -714,13 +718,6 @@ 0 val_0 12 2008-04-08 0 val_0 12 2008-04-08 0 val_0 12 2008-04-08 -2 val_2 12 2008-04-08 -4 val_4 12 2008-04-08 -5 val_5 12 2008-04-08 -5 val_5 12 2008-04-08 -5 val_5 12 2008-04-08 -8 val_8 12 2008-04-08 -9 val_9 12 2008-04-08 10 val_10 12 2008-04-08 11 val_11 12 2008-04-08 12 val_12 12 2008-04-08 @@ -731,6 +728,7 @@ 18 val_18 12 2008-04-08 18 val_18 12 2008-04-08 19 val_19 12 2008-04-08 +2 val_2 12 2008-04-08 20 val_20 12 2008-04-08 24 val_24 12 2008-04-08 24 val_24 12 2008-04-08 @@ -746,12 +744,16 @@ 35 val_35 12 2008-04-08 37 val_37 12 2008-04-08 37 val_37 12 2008-04-08 +4 val_4 12 2008-04-08 41 val_41 12 2008-04-08 42 val_42 12 2008-04-08 42 val_42 12 2008-04-08 43 val_43 12 2008-04-08 44 val_44 12 2008-04-08 47 val_47 12 2008-04-08 +5 val_5 12 2008-04-08 +5 val_5 12 2008-04-08 +5 val_5 12 2008-04-08 51 val_51 12 2008-04-08 51 val_51 12 2008-04-08 53 val_53 12 2008-04-08 @@ -775,6 +777,7 @@ 76 val_76 12 2008-04-08 77 val_77 12 2008-04-08 78 val_78 12 2008-04-08 +8 val_8 12 2008-04-08 80 val_80 12 2008-04-08 82 val_82 12 2008-04-08 83 val_83 12 2008-04-08 @@ -784,6 +787,7 @@ 85 val_85 12 2008-04-08 86 val_86 12 2008-04-08 87 val_87 12 2008-04-08 +9 val_9 12 2008-04-08 90 val_90 12 2008-04-08 90 val_90 12 2008-04-08 90 val_90 12 2008-04-08 @@ -806,13 +810,6 @@ 0 val_0 12 2008-04-09 0 val_0 12 2008-04-09 0 val_0 12 2008-04-09 -2 val_2 12 2008-04-09 -4 val_4 12 2008-04-09 -5 val_5 12 2008-04-09 -5 val_5 12 2008-04-09 -5 val_5 12 2008-04-09 -8 val_8 12 2008-04-09 -9 val_9 12 2008-04-09 10 val_10 12 2008-04-09 11 val_11 12 2008-04-09 12 val_12 12 2008-04-09 @@ -823,6 +820,7 @@ 18 val_18 12 2008-04-09 18 val_18 12 2008-04-09 19 val_19 12 2008-04-09 +2 val_2 12 2008-04-09 20 val_20 12 2008-04-09 24 val_24 12 2008-04-09 24 val_24 12 2008-04-09 @@ -838,12 +836,16 @@ 35 val_35 12 2008-04-09 37 val_37 12 2008-04-09 37 val_37 12 2008-04-09 +4 val_4 12 2008-04-09 41 val_41 12 2008-04-09 42 val_42 12 2008-04-09 42 val_42 12 2008-04-09 43 val_43 12 2008-04-09 44 val_44 12 2008-04-09 47 val_47 12 2008-04-09 +5 val_5 12 2008-04-09 +5 val_5 12 2008-04-09 +5 val_5 12 2008-04-09 51 val_51 12 2008-04-09 51 val_51 12 2008-04-09 53 val_53 12 2008-04-09 @@ -867,6 +869,7 @@ 76 val_76 12 2008-04-09 77 val_77 12 2008-04-09 78 val_78 12 2008-04-09 +8 val_8 12 2008-04-09 80 val_80 12 2008-04-09 82 val_82 12 2008-04-09 83 val_83 12 2008-04-09 @@ -876,6 +879,7 @@ 85 val_85 12 2008-04-09 86 val_86 12 2008-04-09 87 val_87 12 2008-04-09 +9 val_9 12 2008-04-09 90 val_90 12 2008-04-09 90 val_90 12 2008-04-09 90 val_90 12 2008-04-09 Index: ql/src/test/results/clientpositive/join0.q.java1.7.out =================================================================== --- ql/src/test/results/clientpositive/join0.q.java1.7.out (revision 1653459) +++ ql/src/test/results/clientpositive/join0.q.java1.7.out (working copy) @@ -1,5 +1,6 @@ Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, @@ -10,6 +11,7 @@ SORT BY k1, v1, k2, v2 PREHOOK: type: QUERY POSTHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, Index: ql/src/test/results/clientpositive/join15.q.out =================================================================== --- ql/src/test/results/clientpositive/join15.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join15.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key) SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key) SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/join18.q.out =================================================================== --- ql/src/test/results/clientpositive/join18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join18.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT a.key, a.value, b.key, b.value FROM ( @@ -11,7 +13,9 @@ ) b ON (a.key = b.key) PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT a.key, a.value, b.key, b.value FROM ( @@ -176,7 +180,6 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 7 0 3 NULL NULL 10 1 NULL NULL 100 2 NULL NULL @@ -486,3 +489,4 @@ 96 1 NULL NULL 97 2 NULL NULL 98 2 98 1 +NULL NULL 7 Index: ql/src/test/results/clientpositive/join20.q.out =================================================================== --- ql/src/test/results/clientpositive/join20.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join20.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY @@ -111,6 +115,64 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +2 val_2 2 val_2 2 val_2 +4 val_4 4 val_4 4 val_4 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +8 val_8 8 val_8 8 val_8 +9 val_9 9 val_9 9 val_9 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -601,64 +663,6 @@ NULL NULL NULL NULL 97 val_97 NULL NULL NULL NULL 98 val_98 NULL NULL NULL NULL 98 val_98 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -2 val_2 2 val_2 2 val_2 -4 val_4 4 val_4 4 val_4 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -8 val_8 8 val_8 8 val_8 -9 val_9 9 val_9 9 val_9 PREHOOK: query: EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value @@ -772,6 +776,64 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +2 val_2 2 val_2 2 val_2 +4 val_4 4 val_4 4 val_4 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +8 val_8 8 val_8 8 val_8 +9 val_9 9 val_9 9 val_9 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -1262,61 +1324,3 @@ NULL NULL NULL NULL 97 val_97 NULL NULL NULL NULL 98 val_98 NULL NULL NULL NULL 98 val_98 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -2 val_2 2 val_2 2 val_2 -4 val_4 4 val_4 4 val_4 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -8 val_8 8 val_8 8 val_8 -9 val_9 9 val_9 9 val_9 Index: ql/src/test/results/clientpositive/join21.q.out =================================================================== --- ql/src/test/results/clientpositive/join21.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join21.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/join23.q.out =================================================================== --- ql/src/test/results/clientpositive/join23.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join23.q.out (working copy) @@ -1,8 +1,12 @@ Warning: Shuffle Join JOIN[4][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/join6.q.out =================================================================== --- ql/src/test/results/clientpositive/join6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join6.q.out (working copy) @@ -6,7 +6,9 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( @@ -21,7 +23,9 @@ ) c INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( Index: ql/src/test/results/clientpositive/join7.q.out =================================================================== --- ql/src/test/results/clientpositive/join7.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join7.q.out (working copy) @@ -6,7 +6,9 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( @@ -26,7 +28,9 @@ ) c INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( Index: ql/src/test/results/clientpositive/join_array.q.out =================================================================== --- ql/src/test/results/clientpositive/join_array.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join_array.q.out (working copy) @@ -30,11 +30,15 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@tinyb -PREHOOK: query: select * from tinyA +PREHOOK: query: -- SORT_QUERY_RESULTS + +select * from tinyA PREHOOK: type: QUERY PREHOOK: Input: default@tinya #### A masked pattern was here #### -POSTHOOK: query: select * from tinyA +POSTHOOK: query: -- SORT_QUERY_RESULTS + +select * from tinyA POSTHOOK: type: QUERY POSTHOOK: Input: default@tinya #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/join_nullsafe.q.out =================================================================== --- ql/src/test/results/clientpositive/join_nullsafe.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/join_nullsafe.q.out (working copy) @@ -1525,7 +1525,7 @@ Reduce Output Operator key expressions: null (type: void) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: int) TableScan @@ -1537,7 +1537,7 @@ Reduce Output Operator key expressions: null (type: void) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: int) Reduce Operator Tree: Index: ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out =================================================================== --- ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out (working copy) @@ -1,6 +1,10 @@ -PREHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08' +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select ds from srcpart where hr=11 and ds='2008-04-08' PREHOOK: type: QUERY -POSTHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08' +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select ds from srcpart where hr=11 and ds='2008-04-08' POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Index: ql/src/test/results/clientpositive/mapjoin_filter_on_outerjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_filter_on_outerjoin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/mapjoin_filter_on_outerjoin.q.out (working copy) @@ -1,5 +1,7 @@ -PREHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +PREHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + SELECT * FROM src1 RIGHT OUTER JOIN src1 src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 300) @@ -8,8 +10,10 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +POSTHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + SELECT * FROM src1 RIGHT OUTER JOIN src1 src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 300) Index: ql/src/test/results/clientpositive/mapjoin_test_outer.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_test_outer.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/mapjoin_test_outer.q.out (working copy) @@ -1,11 +1,15 @@ -PREHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +PREHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + create table dest_1 (key STRING, value STRING) stored as textfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest_1 -POSTHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +POSTHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + create table dest_1 (key STRING, value STRING) stored as textfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -66,7 +70,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -231,6 +234,7 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 PREHOOK: query: explain SELECT /*+ mapjoin(src1, src2) */ * FROM src1 RIGHT OUTER JOIN dest_1 src2 ON (src1.key = src2.key) @@ -341,7 +345,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -506,6 +509,7 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 PREHOOK: query: SELECT /*+ mapjoin(src1, src2) */ * FROM src1 RIGHT OUTER JOIN dest_1 src2 ON (src1.key = src2.key) JOIN dest_2 src3 ON (src1.key = src3.key) @@ -1162,7 +1166,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -1327,3 +1330,4 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 Index: ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out =================================================================== --- ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out (working copy) @@ -94,6 +94,29 @@ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE ListSink +PREHOOK: query: -- Scans without limit (should be Fetch task now) +explain select concat(key, value) from src +PREHOOK: type: QUERY +POSTHOOK: query: -- Scans without limit (should be Fetch task now) +explain select concat(key, value) from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: concat(key, value) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + PREHOOK: query: -- from HIVE-7397, limit + partition pruning filter explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10 PREHOOK: type: QUERY @@ -125,6 +148,32 @@ POSTHOOK: query: explain select cast(key as int) * 10, upper(value) from src limit 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (UDFToInteger(key) * 10) (type: int), upper(value) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: -- Scans without limit (should not be Fetch task now) +explain select concat(key, value) from src +PREHOOK: type: QUERY +POSTHOOK: query: -- Scans without limit (should not be Fetch task now) +explain select concat(key, value) from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -136,23 +185,20 @@ alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (UDFToInteger(key) * 10) (type: int), upper(value) (type: string) - outputColumnNames: _col0, _col1 + expressions: concat(key, value) (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator - limit: 10 + limit: -1 Processor Tree: ListSink Index: ql/src/test/results/clientpositive/ppd2.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd2.q.out (working copy) @@ -347,7 +347,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -359,7 +359,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/ppd_clusterby.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_clusterby.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd_clusterby.q.out (working copy) @@ -25,7 +25,7 @@ Reduce Output Operator key expressions: '10' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '10' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -80,7 +80,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -92,7 +92,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -174,7 +174,7 @@ Reduce Output Operator key expressions: '10' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '10' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -229,7 +229,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -241,7 +241,7 @@ Reduce Output Operator key expressions: '20' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '20' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/ppd_join4.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_join4.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd_join4.q.out (working copy) @@ -82,7 +82,7 @@ Reduce Output Operator key expressions: 'a' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 'a' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: t3 @@ -93,7 +93,7 @@ Reduce Output Operator key expressions: 'a' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 'a' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/ppd_outer_join5.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_outer_join5.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd_outer_join5.q.out (working copy) @@ -73,9 +73,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Reduce Operator Tree: @@ -126,9 +126,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) TableScan @@ -138,9 +138,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) TableScan @@ -164,10 +164,10 @@ 0 id (type: int) 1 id (type: int) 2 id (type: int) - outputColumnNames: _col1, _col2, _col7, _col8, _col12, _col13, _col14 + outputColumnNames: _col0, _col1, _col2, _col7, _col8, _col12, _col13, _col14 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: 20 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -203,9 +203,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) TableScan @@ -215,9 +215,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) TableScan @@ -241,10 +241,10 @@ 0 id (type: int) 1 id (type: int) 2 id (type: int) - outputColumnNames: _col1, _col2, _col7, _col8, _col12, _col13, _col14 + outputColumnNames: _col0, _col1, _col2, _col7, _col8, _col12, _col13, _col14 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: 20 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator Index: ql/src/test/results/clientpositive/ppd_transform.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_transform.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd_transform.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM src SELECT TRANSFORM(src.key, src.value) @@ -7,7 +9,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM src SELECT TRANSFORM(src.key, src.value) Index: ql/src/test/results/clientpositive/ppd_union_view.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_union_view.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ppd_union_view.q.out (working copy) @@ -179,7 +179,7 @@ Reduce Output Operator key expressions: keymap (type: string), '2011-10-13' (type: string) sort order: ++ - Map-reduce partition columns: keymap (type: string) + Map-reduce partition columns: keymap (type: string), '2011-10-13' (type: string) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) @@ -195,7 +195,7 @@ Reduce Output Operator key expressions: keymap (type: string), '2011-10-13' (type: string) sort order: ++ - Map-reduce partition columns: keymap (type: string) + Map-reduce partition columns: keymap (type: string), '2011-10-13' (type: string) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: string) Index: ql/src/test/results/clientpositive/ptf_matchpath.q.out =================================================================== --- ql/src/test/results/clientpositive/ptf_matchpath.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ptf_matchpath.q.out (working copy) @@ -34,7 +34,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@flights_tiny -PREHOOK: query: -- 1. basic Matchpath test +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,7 +49,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny #### A masked pattern was here #### -POSTHOOK: query: -- 1. basic Matchpath test +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -65,17 +69,17 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 Chicago 1531 2010 10 21 2 21 Chicago 1531 2010 10 25 3 25 Chicago 1531 2010 10 26 2 26 -Baltimore 1599 2010 10 21 2 21 -Baltimore 1599 2010 10 25 3 25 -Baltimore 1599 2010 10 26 2 26 Chicago 361 2010 10 20 2 20 -Washington 7291 2010 10 27 2 27 Chicago 897 2010 10 20 4 20 Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on Index: ql/src/test/results/clientpositive/ptf_rcfile.q.out =================================================================== --- ql/src/test/results/clientpositive/ptf_rcfile.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ptf_rcfile.q.out (working copy) @@ -38,7 +38,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@part_rc -PREHOOK: query: -- testWindowingPTFWithPartRC +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartRC select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, @@ -49,7 +51,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@part_rc #### A masked pattern was here #### -POSTHOOK: query: -- testWindowingPTFWithPartRC +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartRC select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, Index: ql/src/test/results/clientpositive/ptf_register_tblfn.q.out =================================================================== --- ql/src/test/results/clientpositive/ptf_register_tblfn.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ptf_register_tblfn.q.out (working copy) @@ -40,7 +40,9 @@ POSTHOOK: query: create temporary function matchpathtest as 'org.apache.hadoop.hive.ql.udf.ptf.MatchPath$MatchPathResolver' POSTHOOK: type: CREATEFUNCTION POSTHOOK: Output: matchpathtest -PREHOOK: query: -- 1. basic Matchpath test +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpathtest(on flights_tiny @@ -53,7 +55,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny #### A masked pattern was here #### -POSTHOOK: query: -- 1. basic Matchpath test +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpathtest(on flights_tiny @@ -71,17 +75,17 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 Chicago 1531 2010 10 21 2 21 Chicago 1531 2010 10 25 3 25 Chicago 1531 2010 10 26 2 26 -Baltimore 1599 2010 10 21 2 21 -Baltimore 1599 2010 10 25 3 25 -Baltimore 1599 2010 10 26 2 26 Chicago 361 2010 10 20 2 20 -Washington 7291 2010 10 27 2 27 Chicago 897 2010 10 20 4 20 Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 PREHOOK: query: drop temporary function matchpathtest PREHOOK: type: DROPFUNCTION PREHOOK: Output: matchpathtest Index: ql/src/test/results/clientpositive/ptf_seqfile.q.out =================================================================== --- ql/src/test/results/clientpositive/ptf_seqfile.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/ptf_seqfile.q.out (working copy) @@ -38,7 +38,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@part_seq -PREHOOK: query: -- testWindowingPTFWithPartSeqFile +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartSeqFile select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, @@ -49,7 +51,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@part_seq #### A masked pattern was here #### -POSTHOOK: query: -- testWindowingPTFWithPartSeqFile +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartSeqFile select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, Index: ql/src/test/results/clientpositive/quotedid_basic.q.out =================================================================== --- ql/src/test/results/clientpositive/quotedid_basic.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/quotedid_basic.q.out (working copy) @@ -195,9 +195,11 @@ sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -278,9 +280,11 @@ sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Index: ql/src/test/results/clientpositive/regex_col.q.out =================================================================== --- ql/src/test/results/clientpositive/regex_col.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/regex_col.q.out (working copy) @@ -174,7 +174,7 @@ Reduce Output Operator key expressions: '103' (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ - Map-reduce partition columns: _col2 (type: string), _col1 (type: string) + Map-reduce partition columns: '103' (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE TableScan alias: a @@ -189,7 +189,7 @@ Reduce Output Operator key expressions: '103' (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ - Map-reduce partition columns: _col2 (type: string), _col1 (type: string) + Map-reduce partition columns: '103' (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/sample3.q.out =================================================================== --- ql/src/test/results/clientpositive/sample3.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/sample3.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: -- no input pruning, sample filter +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN SELECT s.key FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s PREHOOK: type: QUERY -POSTHOOK: query: -- no input pruning, sample filter +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN SELECT s.key FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s @@ -55,48 +59,8 @@ 0 0 0 -5 -5 -5 -5 10 10 -15 -15 -15 -15 -20 -20 -20 -30 -30 -35 -35 -35 -35 -35 -35 -40 -40 -50 -60 -65 -65 -65 -70 -70 -70 -70 -75 -80 -80 -85 -85 -90 -90 -90 -95 -95 100 100 100 @@ -116,6 +80,10 @@ 135 140 145 +15 +15 +15 +15 150 155 160 @@ -134,6 +102,9 @@ 190 195 195 +20 +20 +20 200 200 205 @@ -167,6 +138,8 @@ 285 295 295 +30 +30 300 300 305 @@ -184,6 +157,12 @@ 335 340 345 +35 +35 +35 +35 +35 +35 355 355 360 @@ -204,6 +183,8 @@ 395 395 395 +40 +40 400 405 410 @@ -239,3 +220,26 @@ 490 495 495 +5 +5 +5 +5 +50 +60 +65 +65 +65 +70 +70 +70 +70 +75 +80 +80 +85 +85 +90 +90 +90 +95 +95 Index: ql/src/test/results/clientpositive/sample5.q.out =================================================================== --- ql/src/test/results/clientpositive/sample5.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/sample5.q.out (working copy) @@ -6,12 +6,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: -- no input pruning, sample filter +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s PREHOOK: type: QUERY -POSTHOOK: query: -- no input pruning, sample filter +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s @@ -349,48 +353,8 @@ 0 val_0 0 val_1 0 val_1 -5 val_5 -5 val_5 -5 val_5 -5 val_6 10 val_10 10 val_11 -15 val_15 -15 val_15 -15 val_16 -15 val_16 -20 val_20 -20 val_21 -20 val_21 -30 val_30 -30 val_31 -35 val_35 -35 val_35 -35 val_35 -35 val_36 -35 val_36 -35 val_36 -40 val_41 -40 val_41 -50 val_51 -60 val_61 -65 val_65 -65 val_66 -65 val_66 -70 val_70 -70 val_70 -70 val_70 -70 val_71 -75 val_76 -80 val_80 -80 val_81 -85 val_85 -85 val_86 -90 val_90 -90 val_90 -90 val_90 -95 val_95 -95 val_95 100 val_100 100 val_100 100 val_101 @@ -410,6 +374,10 @@ 135 val_136 140 val_141 145 val_145 +15 val_15 +15 val_15 +15 val_16 +15 val_16 150 val_150 155 val_155 160 val_160 @@ -428,6 +396,9 @@ 190 val_190 195 val_195 195 val_195 +20 val_20 +20 val_21 +20 val_21 200 val_200 200 val_200 205 val_205 @@ -461,6 +432,8 @@ 285 val_286 295 val_296 295 val_296 +30 val_30 +30 val_31 300 val_301 300 val_301 305 val_305 @@ -478,6 +451,12 @@ 335 val_336 340 val_341 345 val_345 +35 val_35 +35 val_35 +35 val_35 +35 val_36 +35 val_36 +35 val_36 355 val_356 355 val_356 360 val_360 @@ -498,6 +477,8 @@ 395 val_395 395 val_396 395 val_396 +40 val_41 +40 val_41 400 val_400 405 val_406 410 val_411 @@ -533,3 +514,26 @@ 490 val_491 495 val_495 495 val_496 +5 val_5 +5 val_5 +5 val_5 +5 val_6 +50 val_51 +60 val_61 +65 val_65 +65 val_66 +65 val_66 +70 val_70 +70 val_70 +70 val_70 +70 val_71 +75 val_76 +80 val_80 +80 val_81 +85 val_85 +85 val_86 +90 val_90 +90 val_90 +90 val_90 +95 val_95 +95 val_95 Index: ql/src/test/results/clientpositive/scriptfile1.q.out =================================================================== --- ql/src/test/results/clientpositive/scriptfile1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/scriptfile1.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: -- EXCLUDE_OS_WINDOWS +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: query: -- EXCLUDE_OS_WINDOWS +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default Index: ql/src/test/results/clientpositive/selectDistinctStar.q.out =================================================================== --- ql/src/test/results/clientpositive/selectDistinctStar.q.out (revision 0) +++ ql/src/test/results/clientpositive/selectDistinctStar.q.out (working copy) @@ -0,0 +1,4882 @@ +PREHOOK: query: explain select distinct src.* from src +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct src.* from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct src.* from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct src.* from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct * from src where key < '3' +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct * from src where key < '3' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < '3') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct * from src where key < '3' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src where key < '3' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +PREHOOK: query: from src a select distinct a.* where a.key = '238' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: from src a select distinct a.* where a.key = '238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 val_238 +PREHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists sdi +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists sdi +POSTHOOK: type: DROPVIEW +PREHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create View Operator: + Create View + or replace: false + columns: key string, value string + expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 + name: default.sdi + original text: select distinct * from src order by key limit 2 + +PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@sdi +POSTHOOK: query: create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sdi +PREHOOK: query: describe extended sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe extended sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: describe formatted sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe formatted sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select distinct * from src order by key limit 2 +View Expanded Text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 +PREHOOK: query: select * from sdi +PREHOOK: type: QUERY +PREHOOK: Input: default@sdi +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from sdi +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sdi +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +PREHOOK: query: select distinct * from src union all select distinct * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src union all select distinct * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +PREHOOK: query: select distinct * from src join src1 on src.key=src1.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src join src1 on src.key=src1.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +128 128 val_128 +146 val_146 146 val_146 +150 val_150 150 val_150 +213 val_213 213 val_213 +224 224 val_224 +238 val_238 238 val_238 +255 val_255 255 val_255 +273 val_273 273 val_273 +278 val_278 278 val_278 +311 val_311 311 val_311 +369 369 val_369 +401 val_401 401 val_401 +406 val_406 406 val_406 +66 val_66 66 val_66 +98 val_98 98 val_98 +PREHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +146 val_146 2008-04-08 11 146 val_146 146 val_146 +150 val_150 2008-04-08 11 150 val_150 150 val_150 +213 val_213 2008-04-08 11 213 val_213 213 val_213 +238 val_238 2008-04-08 11 238 val_238 238 val_238 +255 val_255 2008-04-08 11 255 val_255 255 val_255 +273 val_273 2008-04-08 11 273 val_273 273 val_273 +278 val_278 2008-04-08 11 278 val_278 278 val_278 +311 val_311 2008-04-08 11 311 val_311 311 val_311 +401 val_401 2008-04-08 11 401 val_401 401 val_401 +406 val_406 2008-04-08 11 406 val_406 406 val_406 +66 val_66 2008-04-08 11 66 val_66 66 val_66 +98 val_98 2008-04-08 11 98 val_98 98 val_98 +PREHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 +103 val_103 103 val_103 +104 val_104 104 val_104 +105 val_105 105 val_105 +11 val_11 11 val_11 +111 val_111 111 val_111 +113 val_113 113 val_113 +114 val_114 114 val_114 +116 val_116 116 val_116 +118 val_118 118 val_118 +119 val_119 119 val_119 +12 val_12 12 val_12 +120 val_120 120 val_120 +125 val_125 125 val_125 +126 val_126 126 val_126 +128 val_128 128 val_128 +129 val_129 129 val_129 +131 val_131 131 val_131 +133 val_133 133 val_133 +134 val_134 134 val_134 +136 val_136 136 val_136 +137 val_137 137 val_137 +138 val_138 138 val_138 +143 val_143 143 val_143 +145 val_145 145 val_145 +146 val_146 146 val_146 +149 val_149 149 val_149 +15 val_15 15 val_15 +150 val_150 150 val_150 +152 val_152 152 val_152 +153 val_153 153 val_153 +155 val_155 155 val_155 +156 val_156 156 val_156 +157 val_157 157 val_157 +158 val_158 158 val_158 +160 val_160 160 val_160 +162 val_162 162 val_162 +163 val_163 163 val_163 +164 val_164 164 val_164 +165 val_165 165 val_165 +166 val_166 166 val_166 +167 val_167 167 val_167 +168 val_168 168 val_168 +169 val_169 169 val_169 +17 val_17 17 val_17 +170 val_170 170 val_170 +172 val_172 172 val_172 +174 val_174 174 val_174 +175 val_175 175 val_175 +176 val_176 176 val_176 +177 val_177 177 val_177 +178 val_178 178 val_178 +179 val_179 179 val_179 +18 val_18 18 val_18 +180 val_180 180 val_180 +181 val_181 181 val_181 +183 val_183 183 val_183 +186 val_186 186 val_186 +187 val_187 187 val_187 +189 val_189 189 val_189 +19 val_19 19 val_19 +190 val_190 190 val_190 +191 val_191 191 val_191 +192 val_192 192 val_192 +193 val_193 193 val_193 +194 val_194 194 val_194 +195 val_195 195 val_195 +196 val_196 196 val_196 +197 val_197 197 val_197 +199 val_199 199 val_199 +2 val_2 2 val_2 +20 val_20 20 val_20 +200 val_200 200 val_200 +201 val_201 201 val_201 +202 val_202 202 val_202 +203 val_203 203 val_203 +205 val_205 205 val_205 +207 val_207 207 val_207 +208 val_208 208 val_208 +209 val_209 209 val_209 +213 val_213 213 val_213 +214 val_214 214 val_214 +216 val_216 216 val_216 +217 val_217 217 val_217 +218 val_218 218 val_218 +219 val_219 219 val_219 +221 val_221 221 val_221 +222 val_222 222 val_222 +223 val_223 223 val_223 +224 val_224 224 val_224 +226 val_226 226 val_226 +228 val_228 228 val_228 +229 val_229 229 val_229 +230 val_230 230 val_230 +233 val_233 233 val_233 +235 val_235 235 val_235 +237 val_237 237 val_237 +238 val_238 238 val_238 +239 val_239 239 val_239 +24 val_24 24 val_24 +241 val_241 241 val_241 +242 val_242 242 val_242 +244 val_244 244 val_244 +247 val_247 247 val_247 +248 val_248 248 val_248 +249 val_249 249 val_249 +252 val_252 252 val_252 +255 val_255 255 val_255 +256 val_256 256 val_256 +257 val_257 257 val_257 +258 val_258 258 val_258 +26 val_26 26 val_26 +260 val_260 260 val_260 +262 val_262 262 val_262 +263 val_263 263 val_263 +265 val_265 265 val_265 +266 val_266 266 val_266 +27 val_27 27 val_27 +272 val_272 272 val_272 +273 val_273 273 val_273 +274 val_274 274 val_274 +275 val_275 275 val_275 +277 val_277 277 val_277 +278 val_278 278 val_278 +28 val_28 28 val_28 +280 val_280 280 val_280 +281 val_281 281 val_281 +282 val_282 282 val_282 +283 val_283 283 val_283 +284 val_284 284 val_284 +285 val_285 285 val_285 +286 val_286 286 val_286 +287 val_287 287 val_287 +288 val_288 288 val_288 +289 val_289 289 val_289 +291 val_291 291 val_291 +292 val_292 292 val_292 +296 val_296 296 val_296 +298 val_298 298 val_298 +30 val_30 30 val_30 +302 val_302 302 val_302 +305 val_305 305 val_305 +306 val_306 306 val_306 +307 val_307 307 val_307 +308 val_308 308 val_308 +309 val_309 309 val_309 +310 val_310 310 val_310 +311 val_311 311 val_311 +315 val_315 315 val_315 +316 val_316 316 val_316 +317 val_317 317 val_317 +318 val_318 318 val_318 +321 val_321 321 val_321 +322 val_322 322 val_322 +323 val_323 323 val_323 +325 val_325 325 val_325 +327 val_327 327 val_327 +33 val_33 33 val_33 +331 val_331 331 val_331 +332 val_332 332 val_332 +333 val_333 333 val_333 +335 val_335 335 val_335 +336 val_336 336 val_336 +338 val_338 338 val_338 +339 val_339 339 val_339 +34 val_34 34 val_34 +341 val_341 341 val_341 +342 val_342 342 val_342 +344 val_344 344 val_344 +345 val_345 345 val_345 +348 val_348 348 val_348 +35 val_35 35 val_35 +351 val_351 351 val_351 +353 val_353 353 val_353 +356 val_356 356 val_356 +360 val_360 360 val_360 +362 val_362 362 val_362 +364 val_364 364 val_364 +365 val_365 365 val_365 +366 val_366 366 val_366 +367 val_367 367 val_367 +368 val_368 368 val_368 +369 val_369 369 val_369 +37 val_37 37 val_37 +373 val_373 373 val_373 +374 val_374 374 val_374 +375 val_375 375 val_375 +377 val_377 377 val_377 +378 val_378 378 val_378 +379 val_379 379 val_379 +382 val_382 382 val_382 +384 val_384 384 val_384 +386 val_386 386 val_386 +389 val_389 389 val_389 +392 val_392 392 val_392 +393 val_393 393 val_393 +394 val_394 394 val_394 +395 val_395 395 val_395 +396 val_396 396 val_396 +397 val_397 397 val_397 +399 val_399 399 val_399 +4 val_4 4 val_4 +400 val_400 400 val_400 +401 val_401 401 val_401 +402 val_402 402 val_402 +403 val_403 403 val_403 +404 val_404 404 val_404 +406 val_406 406 val_406 +407 val_407 407 val_407 +409 val_409 409 val_409 +41 val_41 41 val_41 +411 val_411 411 val_411 +413 val_413 413 val_413 +414 val_414 414 val_414 +417 val_417 417 val_417 +418 val_418 418 val_418 +419 val_419 419 val_419 +42 val_42 42 val_42 +421 val_421 421 val_421 +424 val_424 424 val_424 +427 val_427 427 val_427 +429 val_429 429 val_429 +43 val_43 43 val_43 +430 val_430 430 val_430 +431 val_431 431 val_431 +432 val_432 432 val_432 +435 val_435 435 val_435 +436 val_436 436 val_436 +437 val_437 437 val_437 +438 val_438 438 val_438 +439 val_439 439 val_439 +44 val_44 44 val_44 +443 val_443 443 val_443 +444 val_444 444 val_444 +446 val_446 446 val_446 +448 val_448 448 val_448 +449 val_449 449 val_449 +452 val_452 452 val_452 +453 val_453 453 val_453 +454 val_454 454 val_454 +455 val_455 455 val_455 +457 val_457 457 val_457 +458 val_458 458 val_458 +459 val_459 459 val_459 +460 val_460 460 val_460 +462 val_462 462 val_462 +463 val_463 463 val_463 +466 val_466 466 val_466 +467 val_467 467 val_467 +468 val_468 468 val_468 +469 val_469 469 val_469 +47 val_47 47 val_47 +470 val_470 470 val_470 +472 val_472 472 val_472 +475 val_475 475 val_475 +477 val_477 477 val_477 +478 val_478 478 val_478 +479 val_479 479 val_479 +480 val_480 480 val_480 +481 val_481 481 val_481 +482 val_482 482 val_482 +483 val_483 483 val_483 +484 val_484 484 val_484 +485 val_485 485 val_485 +487 val_487 487 val_487 +489 val_489 489 val_489 +490 val_490 490 val_490 +491 val_491 491 val_491 +492 val_492 492 val_492 +493 val_493 493 val_493 +494 val_494 494 val_494 +495 val_495 495 val_495 +496 val_496 496 val_496 +497 val_497 497 val_497 +498 val_498 498 val_498 +5 val_5 5 val_5 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +8 val_8 8 val_8 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +9 val_9 9 val_9 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +98 val_98 98 val_98 +PREHOOK: query: select distinct * from (select distinct * from src)src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from (select distinct * from src)src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct src.* from src +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct src.* from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct src.* from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct src.* from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct * from src where key < '3' +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct * from src where key < '3' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < '3') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct * from src where key < '3' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src where key < '3' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +PREHOOK: query: from src a select distinct a.* where a.key = '238' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: from src a select distinct a.* where a.key = '238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 val_238 +PREHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists sdi +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@sdi +PREHOOK: Output: default@sdi +POSTHOOK: query: drop view if exists sdi +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@sdi +POSTHOOK: Output: default@sdi +PREHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create View Operator: + Create View + or replace: false + columns: key string, value string + expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 + name: default.sdi + original text: select distinct * from src order by key limit 2 + +PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@sdi +POSTHOOK: query: create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sdi +PREHOOK: query: describe extended sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe extended sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: describe formatted sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe formatted sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select distinct * from src order by key limit 2 +View Expanded Text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 +PREHOOK: query: select * from sdi +PREHOOK: type: QUERY +PREHOOK: Input: default@sdi +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from sdi +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sdi +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +PREHOOK: query: select distinct * from src union all select distinct * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src union all select distinct * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +PREHOOK: query: select distinct * from src join src1 on src.key=src1.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src join src1 on src.key=src1.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +128 128 val_128 +146 val_146 146 val_146 +150 val_150 150 val_150 +213 val_213 213 val_213 +224 224 val_224 +238 val_238 238 val_238 +255 val_255 255 val_255 +273 val_273 273 val_273 +278 val_278 278 val_278 +311 val_311 311 val_311 +369 369 val_369 +401 val_401 401 val_401 +406 val_406 406 val_406 +66 val_66 66 val_66 +98 val_98 98 val_98 +PREHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +146 val_146 2008-04-08 11 146 val_146 146 val_146 +150 val_150 2008-04-08 11 150 val_150 150 val_150 +213 val_213 2008-04-08 11 213 val_213 213 val_213 +238 val_238 2008-04-08 11 238 val_238 238 val_238 +255 val_255 2008-04-08 11 255 val_255 255 val_255 +273 val_273 2008-04-08 11 273 val_273 273 val_273 +278 val_278 2008-04-08 11 278 val_278 278 val_278 +311 val_311 2008-04-08 11 311 val_311 311 val_311 +401 val_401 2008-04-08 11 401 val_401 401 val_401 +406 val_406 2008-04-08 11 406 val_406 406 val_406 +66 val_66 2008-04-08 11 66 val_66 66 val_66 +98 val_98 2008-04-08 11 98 val_98 98 val_98 +PREHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 +103 val_103 103 val_103 +104 val_104 104 val_104 +105 val_105 105 val_105 +11 val_11 11 val_11 +111 val_111 111 val_111 +113 val_113 113 val_113 +114 val_114 114 val_114 +116 val_116 116 val_116 +118 val_118 118 val_118 +119 val_119 119 val_119 +12 val_12 12 val_12 +120 val_120 120 val_120 +125 val_125 125 val_125 +126 val_126 126 val_126 +128 val_128 128 val_128 +129 val_129 129 val_129 +131 val_131 131 val_131 +133 val_133 133 val_133 +134 val_134 134 val_134 +136 val_136 136 val_136 +137 val_137 137 val_137 +138 val_138 138 val_138 +143 val_143 143 val_143 +145 val_145 145 val_145 +146 val_146 146 val_146 +149 val_149 149 val_149 +15 val_15 15 val_15 +150 val_150 150 val_150 +152 val_152 152 val_152 +153 val_153 153 val_153 +155 val_155 155 val_155 +156 val_156 156 val_156 +157 val_157 157 val_157 +158 val_158 158 val_158 +160 val_160 160 val_160 +162 val_162 162 val_162 +163 val_163 163 val_163 +164 val_164 164 val_164 +165 val_165 165 val_165 +166 val_166 166 val_166 +167 val_167 167 val_167 +168 val_168 168 val_168 +169 val_169 169 val_169 +17 val_17 17 val_17 +170 val_170 170 val_170 +172 val_172 172 val_172 +174 val_174 174 val_174 +175 val_175 175 val_175 +176 val_176 176 val_176 +177 val_177 177 val_177 +178 val_178 178 val_178 +179 val_179 179 val_179 +18 val_18 18 val_18 +180 val_180 180 val_180 +181 val_181 181 val_181 +183 val_183 183 val_183 +186 val_186 186 val_186 +187 val_187 187 val_187 +189 val_189 189 val_189 +19 val_19 19 val_19 +190 val_190 190 val_190 +191 val_191 191 val_191 +192 val_192 192 val_192 +193 val_193 193 val_193 +194 val_194 194 val_194 +195 val_195 195 val_195 +196 val_196 196 val_196 +197 val_197 197 val_197 +199 val_199 199 val_199 +2 val_2 2 val_2 +20 val_20 20 val_20 +200 val_200 200 val_200 +201 val_201 201 val_201 +202 val_202 202 val_202 +203 val_203 203 val_203 +205 val_205 205 val_205 +207 val_207 207 val_207 +208 val_208 208 val_208 +209 val_209 209 val_209 +213 val_213 213 val_213 +214 val_214 214 val_214 +216 val_216 216 val_216 +217 val_217 217 val_217 +218 val_218 218 val_218 +219 val_219 219 val_219 +221 val_221 221 val_221 +222 val_222 222 val_222 +223 val_223 223 val_223 +224 val_224 224 val_224 +226 val_226 226 val_226 +228 val_228 228 val_228 +229 val_229 229 val_229 +230 val_230 230 val_230 +233 val_233 233 val_233 +235 val_235 235 val_235 +237 val_237 237 val_237 +238 val_238 238 val_238 +239 val_239 239 val_239 +24 val_24 24 val_24 +241 val_241 241 val_241 +242 val_242 242 val_242 +244 val_244 244 val_244 +247 val_247 247 val_247 +248 val_248 248 val_248 +249 val_249 249 val_249 +252 val_252 252 val_252 +255 val_255 255 val_255 +256 val_256 256 val_256 +257 val_257 257 val_257 +258 val_258 258 val_258 +26 val_26 26 val_26 +260 val_260 260 val_260 +262 val_262 262 val_262 +263 val_263 263 val_263 +265 val_265 265 val_265 +266 val_266 266 val_266 +27 val_27 27 val_27 +272 val_272 272 val_272 +273 val_273 273 val_273 +274 val_274 274 val_274 +275 val_275 275 val_275 +277 val_277 277 val_277 +278 val_278 278 val_278 +28 val_28 28 val_28 +280 val_280 280 val_280 +281 val_281 281 val_281 +282 val_282 282 val_282 +283 val_283 283 val_283 +284 val_284 284 val_284 +285 val_285 285 val_285 +286 val_286 286 val_286 +287 val_287 287 val_287 +288 val_288 288 val_288 +289 val_289 289 val_289 +291 val_291 291 val_291 +292 val_292 292 val_292 +296 val_296 296 val_296 +298 val_298 298 val_298 +30 val_30 30 val_30 +302 val_302 302 val_302 +305 val_305 305 val_305 +306 val_306 306 val_306 +307 val_307 307 val_307 +308 val_308 308 val_308 +309 val_309 309 val_309 +310 val_310 310 val_310 +311 val_311 311 val_311 +315 val_315 315 val_315 +316 val_316 316 val_316 +317 val_317 317 val_317 +318 val_318 318 val_318 +321 val_321 321 val_321 +322 val_322 322 val_322 +323 val_323 323 val_323 +325 val_325 325 val_325 +327 val_327 327 val_327 +33 val_33 33 val_33 +331 val_331 331 val_331 +332 val_332 332 val_332 +333 val_333 333 val_333 +335 val_335 335 val_335 +336 val_336 336 val_336 +338 val_338 338 val_338 +339 val_339 339 val_339 +34 val_34 34 val_34 +341 val_341 341 val_341 +342 val_342 342 val_342 +344 val_344 344 val_344 +345 val_345 345 val_345 +348 val_348 348 val_348 +35 val_35 35 val_35 +351 val_351 351 val_351 +353 val_353 353 val_353 +356 val_356 356 val_356 +360 val_360 360 val_360 +362 val_362 362 val_362 +364 val_364 364 val_364 +365 val_365 365 val_365 +366 val_366 366 val_366 +367 val_367 367 val_367 +368 val_368 368 val_368 +369 val_369 369 val_369 +37 val_37 37 val_37 +373 val_373 373 val_373 +374 val_374 374 val_374 +375 val_375 375 val_375 +377 val_377 377 val_377 +378 val_378 378 val_378 +379 val_379 379 val_379 +382 val_382 382 val_382 +384 val_384 384 val_384 +386 val_386 386 val_386 +389 val_389 389 val_389 +392 val_392 392 val_392 +393 val_393 393 val_393 +394 val_394 394 val_394 +395 val_395 395 val_395 +396 val_396 396 val_396 +397 val_397 397 val_397 +399 val_399 399 val_399 +4 val_4 4 val_4 +400 val_400 400 val_400 +401 val_401 401 val_401 +402 val_402 402 val_402 +403 val_403 403 val_403 +404 val_404 404 val_404 +406 val_406 406 val_406 +407 val_407 407 val_407 +409 val_409 409 val_409 +41 val_41 41 val_41 +411 val_411 411 val_411 +413 val_413 413 val_413 +414 val_414 414 val_414 +417 val_417 417 val_417 +418 val_418 418 val_418 +419 val_419 419 val_419 +42 val_42 42 val_42 +421 val_421 421 val_421 +424 val_424 424 val_424 +427 val_427 427 val_427 +429 val_429 429 val_429 +43 val_43 43 val_43 +430 val_430 430 val_430 +431 val_431 431 val_431 +432 val_432 432 val_432 +435 val_435 435 val_435 +436 val_436 436 val_436 +437 val_437 437 val_437 +438 val_438 438 val_438 +439 val_439 439 val_439 +44 val_44 44 val_44 +443 val_443 443 val_443 +444 val_444 444 val_444 +446 val_446 446 val_446 +448 val_448 448 val_448 +449 val_449 449 val_449 +452 val_452 452 val_452 +453 val_453 453 val_453 +454 val_454 454 val_454 +455 val_455 455 val_455 +457 val_457 457 val_457 +458 val_458 458 val_458 +459 val_459 459 val_459 +460 val_460 460 val_460 +462 val_462 462 val_462 +463 val_463 463 val_463 +466 val_466 466 val_466 +467 val_467 467 val_467 +468 val_468 468 val_468 +469 val_469 469 val_469 +47 val_47 47 val_47 +470 val_470 470 val_470 +472 val_472 472 val_472 +475 val_475 475 val_475 +477 val_477 477 val_477 +478 val_478 478 val_478 +479 val_479 479 val_479 +480 val_480 480 val_480 +481 val_481 481 val_481 +482 val_482 482 val_482 +483 val_483 483 val_483 +484 val_484 484 val_484 +485 val_485 485 val_485 +487 val_487 487 val_487 +489 val_489 489 val_489 +490 val_490 490 val_490 +491 val_491 491 val_491 +492 val_492 492 val_492 +493 val_493 493 val_493 +494 val_494 494 val_494 +495 val_495 495 val_495 +496 val_496 496 val_496 +497 val_497 497 val_497 +498 val_498 498 val_498 +5 val_5 5 val_5 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +8 val_8 8 val_8 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +9 val_9 9 val_9 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +98 val_98 98 val_98 +PREHOOK: query: select distinct * from (select distinct * from src)src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from (select distinct * from src)src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 Index: ql/src/test/results/clientpositive/semijoin.q.out =================================================================== --- ql/src/test/results/clientpositive/semijoin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/semijoin.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: create table t1 as select cast(key as int) key, value from src where key <= 10 +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table t1 as select cast(key as int) key, value from src where key <= 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1 as select cast(key as int) key, value from src where key <= 10 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table t1 as select cast(key as int) key, value from src where key <= 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -19,6 +23,7 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 2 val_2 4 val_4 5 val_5 @@ -26,7 +31,6 @@ 5 val_5 8 val_8 9 val_9 -10 val_10 PREHOOK: query: create table t2 as select cast(2*key as int) key, value from t1 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@t1 @@ -48,14 +52,14 @@ 0 val_0 0 val_0 0 val_0 -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 16 val_8 18 val_9 20 val_10 +4 val_2 +8 val_4 PREHOOK: query: create table t3 as select * from (select * from t1 union all select * from t2) b PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@t1 @@ -82,7 +86,14 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 +10 val_5 +10 val_5 +10 val_5 +16 val_8 +18 val_9 2 val_2 +20 val_10 4 val_2 4 val_4 5 val_5 @@ -91,13 +102,6 @@ 8 val_4 8 val_8 9 val_9 -10 val_10 -10 val_5 -10 val_5 -10 val_5 -16 val_8 -18 val_9 -20 val_10 PREHOOK: query: create table t4 (key int, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -215,9 +219,9 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 4 val_4 8 val_8 -10 val_10 PREHOOK: query: explain select * from t2 a left semi join t1 b on b.key=a.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t2 a left semi join t1 b on b.key=a.key sort by a.key, a.value @@ -319,11 +323,11 @@ 0 val_0 0 val_0 0 val_0 -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 +4 val_2 +8 val_4 PREHOOK: query: explain select * from t1 a left semi join t4 b on b.key=a.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 a left semi join t4 b on b.key=a.key sort by a.key, a.value @@ -937,11 +941,11 @@ POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 +4 val_2 +8 val_4 PREHOOK: query: explain select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key @@ -1038,6 +1042,10 @@ 0 0 0 +10 +10 +10 +10 2 4 4 @@ -1047,10 +1055,6 @@ 8 8 9 -10 -10 -10 -10 PREHOOK: query: explain select * from t1 a left semi join t2 b on a.key = 2*b.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 a left semi join t2 b on a.key = 2*b.key sort by a.key, a.value @@ -1281,11 +1285,11 @@ 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 -4 val_4 4 val_2 -8 val_8 8 val_4 10 val_10 10 val_5 10 val_10 10 val_5 10 val_10 10 val_5 +4 val_4 4 val_2 +8 val_8 8 val_4 PREHOOK: query: explain select * from t3 a left semi join t1 b on a.key = b.key and a.value=b.value sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t3 a left semi join t1 b on a.key = b.key and a.value=b.value sort by a.key, a.value @@ -1389,6 +1393,7 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 2 val_2 4 val_4 5 val_5 @@ -1396,7 +1401,6 @@ 5 val_5 8 val_8 9 val_9 -10 val_10 PREHOOK: query: explain select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key @@ -1522,14 +1526,14 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 PREHOOK: query: explain select a.key from t3 a left outer join t1 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left outer join t1 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1651,14 +1655,14 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 PREHOOK: query: explain select a.key from t1 a right outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t1 a right outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1762,9 +1766,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -1783,14 +1784,17 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t1 a full outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t1 a full outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1894,9 +1898,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -1915,14 +1916,17 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.key = c.key sort by a.key @@ -2044,10 +2048,6 @@ 0 0 0 -4 -4 -8 -8 10 10 10 @@ -2055,6 +2055,10 @@ 16 18 20 +4 +4 +8 +8 PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key right outer join t1 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key right outer join t1 c on a.key = c.key sort by a.key @@ -2158,11 +2162,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL 0 0 0 @@ -2181,14 +2180,19 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t1 b on a.key = b.key full outer join t2 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t1 b on a.key = b.key full outer join t2 c on a.key = c.key sort by a.key @@ -2292,9 +2296,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -2313,15 +2314,6 @@ 0 0 0 -2 -4 -4 -5 -5 -5 -8 -8 -9 10 10 10 @@ -2334,6 +2326,18 @@ 10 10 10 +2 +4 +4 +5 +5 +5 +8 +8 +9 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key @@ -2487,10 +2491,6 @@ 0 0 0 -4 -4 -8 -8 10 10 10 @@ -2504,6 +2504,10 @@ 16 18 20 +4 +4 +8 +8 PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value where a.key > 100 PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value where a.key > 100 Index: ql/src/test/results/clientpositive/show_functions.q.out =================================================================== --- ql/src/test/results/clientpositive/show_functions.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/show_functions.q.out (working copy) @@ -21,6 +21,7 @@ ^ abs acos +add_months and array array_contains @@ -86,6 +87,7 @@ in in_file index +initcap inline instr isnotnull @@ -93,6 +95,7 @@ java_method json_tuple lag +last_day last_value lcase lead Index: ql/src/test/results/clientpositive/smb_mapjoin_11.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin_11.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/smb_mapjoin_11.q.out (working copy) @@ -277,6 +277,1959 @@ POSTHOOK: Output: default@test_table3@ds=1 POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM test_table1 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: SELECT * FROM test_table3 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_table1 + TOK_TABLEBUCKETSAMPLE + 2 + 16 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_table1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: 000001_0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Truncated Path -> Alias: + /test_table1/ds=1/000001_0 [test_table1] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_table3 + TOK_TABLEBUCKETSAMPLE + 2 + 16 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_table3 + Statistics: Num rows: 1028 Data size: 10968 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: (ds%3D1)000001_0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 1028 + partition_columns ds + partition_columns.types string + rawDataSize 10968 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11996 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + name: default.test_table3 + Truncated Path -> Alias: + /test_table3/ds=1/(ds%3D1)000001_0 [test_table3] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +17 val_17 1 +33 val_33 1 +65 val_65 1 +97 val_97 1 +97 val_97 1 +113 val_113 1 +113 val_113 1 +129 val_129 1 +129 val_129 1 +145 val_145 1 +177 val_177 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +209 val_209 1 +209 val_209 1 +241 val_241 1 +257 val_257 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +289 val_289 1 +305 val_305 1 +321 val_321 1 +321 val_321 1 +353 val_353 1 +353 val_353 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +449 val_449 1 +481 val_481 1 +497 val_497 1 +PREHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +17 val_17 1 +33 val_33 1 +65 val_65 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +145 val_145 1 +177 val_177 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +241 val_241 1 +257 val_257 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +289 val_289 1 +305 val_305 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +449 val_449 1 +481 val_481 1 +497 val_497 1 PREHOOK: query: -- Join data from a sampled bucket to verify the data is bucketed SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/smb_mapjoin_25.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin_25.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/smb_mapjoin_25.q.out (working copy) @@ -71,7 +71,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE TableScan alias: b @@ -82,7 +82,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -154,7 +154,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: d @@ -165,7 +165,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/sort.q.out =================================================================== --- ql/src/test/results/clientpositive/sort.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/sort.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT x.* FROM SRC x SORT BY key PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT x.* FROM SRC x SORT BY key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/auto_join21.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/auto_join21.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/auto_join21.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/auto_join23.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/auto_join23.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/auto_join23.q.out (working copy) @@ -1,8 +1,12 @@ Warning: Map Join MAPJOIN[12][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/groupby2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/groupby2.q.out (working copy) @@ -87,11 +87,15 @@ POSTHOOK: Lineage: dest_g2.c1 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.c2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT dest_g2.* FROM dest_g2 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 PREHOOK: type: QUERY PREHOOK: Input: default@dest_g2 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest_g2.* FROM dest_g2 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_g2 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/spark/groupby_cube1.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/groupby_cube1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/groupby_cube1.q.out (working copy) @@ -62,8 +62,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -241,8 +242,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -334,8 +336,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -423,8 +426,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: complete - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -562,8 +566,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 @@ -596,8 +601,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out (working copy) @@ -14,12 +14,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest_g3 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT src.key), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1) PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) INSERT OVERWRITE TABLE dest_g3 SELECT substr(src.key,1,1), count(DISTINCT src.key), count(src.value) WHERE substr(src.key,1,1) < 5 GROUP BY substr(src.key,1,1) Index: ql/src/test/results/clientpositive/spark/groupby_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/groupby_ppr.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/groupby_ppr.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 Index: ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out (working copy) @@ -62,8 +62,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -145,8 +146,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -238,8 +240,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 @@ -321,8 +324,9 @@ aggregations: count(DISTINCT KEY._col2:0._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: complete - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -460,8 +464,9 @@ aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 @@ -494,8 +499,9 @@ aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + pruneGroupingSetId: true Select Operator expressions: _col0 (type: string), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 Index: ql/src/test/results/clientpositive/spark/input14.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/input14.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/input14.q.out (working copy) @@ -111,11 +111,15 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/spark/input17.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/input17.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/input17.q.out (working copy) @@ -108,15 +108,18 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### -NULL NULL -1461153966 {"myint":49,"mystring":"343","underscore_int":7} -1952710705 {"myint":25,"mystring":"125","underscore_int":5} -734328905 {"myint":16,"mystring":"64","underscore_int":4} @@ -127,3 +130,4 @@ 336964422 {"myint":81,"mystring":"729","underscore_int":9} 465985201 {"myint":1,"mystring":"1","underscore_int":1} 477111225 {"myint":9,"mystring":"27","underscore_int":3} +NULL NULL Index: ql/src/test/results/clientpositive/spark/input18.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/input18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/input18.q.out (working copy) @@ -111,11 +111,15 @@ POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/spark/input_part2.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/input_part2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/input_part2.q.out (working copy) @@ -14,12 +14,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest2 -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' INSERT OVERWRITE TABLE dest2 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-09' and srcpart.hr = '12' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' INSERT OVERWRITE TABLE dest2 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-09' and srcpart.hr = '12' @@ -401,13 +405,6 @@ 0 val_0 12 2008-04-08 0 val_0 12 2008-04-08 0 val_0 12 2008-04-08 -2 val_2 12 2008-04-08 -4 val_4 12 2008-04-08 -5 val_5 12 2008-04-08 -5 val_5 12 2008-04-08 -5 val_5 12 2008-04-08 -8 val_8 12 2008-04-08 -9 val_9 12 2008-04-08 10 val_10 12 2008-04-08 11 val_11 12 2008-04-08 12 val_12 12 2008-04-08 @@ -418,6 +415,7 @@ 18 val_18 12 2008-04-08 18 val_18 12 2008-04-08 19 val_19 12 2008-04-08 +2 val_2 12 2008-04-08 20 val_20 12 2008-04-08 24 val_24 12 2008-04-08 24 val_24 12 2008-04-08 @@ -433,12 +431,16 @@ 35 val_35 12 2008-04-08 37 val_37 12 2008-04-08 37 val_37 12 2008-04-08 +4 val_4 12 2008-04-08 41 val_41 12 2008-04-08 42 val_42 12 2008-04-08 42 val_42 12 2008-04-08 43 val_43 12 2008-04-08 44 val_44 12 2008-04-08 47 val_47 12 2008-04-08 +5 val_5 12 2008-04-08 +5 val_5 12 2008-04-08 +5 val_5 12 2008-04-08 51 val_51 12 2008-04-08 51 val_51 12 2008-04-08 53 val_53 12 2008-04-08 @@ -462,6 +464,7 @@ 76 val_76 12 2008-04-08 77 val_77 12 2008-04-08 78 val_78 12 2008-04-08 +8 val_8 12 2008-04-08 80 val_80 12 2008-04-08 82 val_82 12 2008-04-08 83 val_83 12 2008-04-08 @@ -471,6 +474,7 @@ 85 val_85 12 2008-04-08 86 val_86 12 2008-04-08 87 val_87 12 2008-04-08 +9 val_9 12 2008-04-08 90 val_90 12 2008-04-08 90 val_90 12 2008-04-08 90 val_90 12 2008-04-08 @@ -493,13 +497,6 @@ 0 val_0 12 2008-04-09 0 val_0 12 2008-04-09 0 val_0 12 2008-04-09 -2 val_2 12 2008-04-09 -4 val_4 12 2008-04-09 -5 val_5 12 2008-04-09 -5 val_5 12 2008-04-09 -5 val_5 12 2008-04-09 -8 val_8 12 2008-04-09 -9 val_9 12 2008-04-09 10 val_10 12 2008-04-09 11 val_11 12 2008-04-09 12 val_12 12 2008-04-09 @@ -510,6 +507,7 @@ 18 val_18 12 2008-04-09 18 val_18 12 2008-04-09 19 val_19 12 2008-04-09 +2 val_2 12 2008-04-09 20 val_20 12 2008-04-09 24 val_24 12 2008-04-09 24 val_24 12 2008-04-09 @@ -525,12 +523,16 @@ 35 val_35 12 2008-04-09 37 val_37 12 2008-04-09 37 val_37 12 2008-04-09 +4 val_4 12 2008-04-09 41 val_41 12 2008-04-09 42 val_42 12 2008-04-09 42 val_42 12 2008-04-09 43 val_43 12 2008-04-09 44 val_44 12 2008-04-09 47 val_47 12 2008-04-09 +5 val_5 12 2008-04-09 +5 val_5 12 2008-04-09 +5 val_5 12 2008-04-09 51 val_51 12 2008-04-09 51 val_51 12 2008-04-09 53 val_53 12 2008-04-09 @@ -554,6 +556,7 @@ 76 val_76 12 2008-04-09 77 val_77 12 2008-04-09 78 val_78 12 2008-04-09 +8 val_8 12 2008-04-09 80 val_80 12 2008-04-09 82 val_82 12 2008-04-09 83 val_83 12 2008-04-09 @@ -563,6 +566,7 @@ 85 val_85 12 2008-04-09 86 val_86 12 2008-04-09 87 val_87 12 2008-04-09 +9 val_9 12 2008-04-09 90 val_90 12 2008-04-09 90 val_90 12 2008-04-09 90 val_90 12 2008-04-09 Index: ql/src/test/results/clientpositive/spark/join0.q.java1.7.out =================================================================== --- ql/src/test/results/clientpositive/spark/join0.q.java1.7.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join0.q.java1.7.out (working copy) @@ -1,5 +1,6 @@ Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Work 'Reducer 2' is a cross product PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, @@ -10,6 +11,7 @@ SORT BY k1, v1, k2, v2 PREHOOK: type: QUERY POSTHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, Index: ql/src/test/results/clientpositive/spark/join15.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join15.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join15.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key) SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key) SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/join18.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join18.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT a.key, a.value, b.key, b.value FROM ( @@ -11,7 +13,9 @@ ) b ON (a.key = b.key) PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT a.key, a.value, b.key, b.value FROM ( @@ -164,7 +168,6 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 7 0 3 NULL NULL 10 1 NULL NULL 100 2 NULL NULL @@ -474,3 +477,4 @@ 96 1 NULL NULL 97 2 NULL NULL 98 2 98 1 +NULL NULL 7 Index: ql/src/test/results/clientpositive/spark/join20.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join20.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join20.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY @@ -111,6 +115,64 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +2 val_2 2 val_2 2 val_2 +4 val_4 4 val_4 4 val_4 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +8 val_8 8 val_8 8 val_8 +9 val_9 9 val_9 9 val_9 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -601,64 +663,6 @@ NULL NULL NULL NULL 97 val_97 NULL NULL NULL NULL 98 val_98 NULL NULL NULL NULL 98 val_98 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -2 val_2 2 val_2 2 val_2 -4 val_4 4 val_4 4 val_4 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -8 val_8 8 val_8 8 val_8 -9 val_9 9 val_9 9 val_9 PREHOOK: query: EXPLAIN SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value @@ -772,6 +776,64 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +0 val_0 0 val_0 0 val_0 +2 val_2 2 val_2 2 val_2 +4 val_4 4 val_4 4 val_4 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +5 val_5 5 val_5 5 val_5 +8 val_8 8 val_8 8 val_8 +9 val_9 9 val_9 9 val_9 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -1262,61 +1324,3 @@ NULL NULL NULL NULL 97 val_97 NULL NULL NULL NULL 98 val_98 NULL NULL NULL NULL 98 val_98 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -0 val_0 0 val_0 0 val_0 -2 val_2 2 val_2 2 val_2 -4 val_4 4 val_4 4 val_4 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -5 val_5 5 val_5 5 val_5 -8 val_8 8 val_8 8 val_8 -9 val_9 9 val_9 9 val_9 Index: ql/src/test/results/clientpositive/spark/join21.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join21.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join21.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/join23.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join23.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join23.q.out (working copy) @@ -1,8 +1,12 @@ Warning: Shuffle Join JOIN[4][tables = [src1, src2]] in Work 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/join6.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join6.q.out (working copy) @@ -6,7 +6,9 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( @@ -21,7 +23,9 @@ ) c INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( Index: ql/src/test/results/clientpositive/spark/join7.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join7.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join7.q.out (working copy) @@ -6,7 +6,9 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( @@ -26,7 +28,9 @@ ) c INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM ( Index: ql/src/test/results/clientpositive/spark/join_array.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/join_array.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/join_array.q.out (working copy) @@ -30,11 +30,15 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@tinyb -PREHOOK: query: select * from tinyA +PREHOOK: query: -- SORT_QUERY_RESULTS + +select * from tinyA PREHOOK: type: QUERY PREHOOK: Input: default@tinya #### A masked pattern was here #### -POSTHOOK: query: select * from tinyA +POSTHOOK: query: -- SORT_QUERY_RESULTS + +select * from tinyA POSTHOOK: type: QUERY POSTHOOK: Input: default@tinya #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out (working copy) @@ -1,6 +1,10 @@ -PREHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08' +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select ds from srcpart where hr=11 and ds='2008-04-08' PREHOOK: type: QUERY -POSTHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08' +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select ds from srcpart where hr=11 and ds='2008-04-08' POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Index: ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out (working copy) @@ -381,119 +381,3 @@ 9 9 9 9 9 9 -PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec) -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -#### A masked pattern was here #### -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -45 45 -45 45 -45 45 -45 45 -45 45 -6 6 -6 6 -6 6 -6 6 -6 6 -6 6 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -70 70 -70 70 -70 70 -70 70 -70 70 -70 70 -70 70 -79 79 -79 79 -79 79 -79 79 -79 79 -79 79 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 Index: ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out (working copy) @@ -1,5 +1,7 @@ -PREHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +PREHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + SELECT * FROM src1 RIGHT OUTER JOIN src1 src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 300) @@ -8,8 +10,10 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +POSTHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + SELECT * FROM src1 RIGHT OUTER JOIN src1 src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 300) Index: ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out (working copy) @@ -1,11 +1,15 @@ -PREHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +PREHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + create table dest_1 (key STRING, value STRING) stored as textfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest_1 -POSTHOOK: query: --HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition +POSTHOOK: query: -- SORT_QUERY_RESULTS +--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition + create table dest_1 (key STRING, value STRING) stored as textfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -66,7 +70,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -231,6 +234,7 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 PREHOOK: query: explain SELECT /*+ mapjoin(src1, src2) */ * FROM src1 RIGHT OUTER JOIN dest_1 src2 ON (src1.key = src2.key) @@ -350,7 +354,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -515,6 +518,7 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 PREHOOK: query: SELECT /*+ mapjoin(src1, src2) */ * FROM src1 RIGHT OUTER JOIN dest_1 src2 ON (src1.key = src2.key) JOIN dest_2 src3 ON (src1.key = src3.key) @@ -1180,7 +1184,6 @@ POSTHOOK: Input: default@dest_2 POSTHOOK: Input: default@src1 #### A masked pattern was here #### -NULL NULL 333444 555666 333444 555666 @@ -1345,3 +1348,4 @@ 146 val_146 146 val_146 146 val_146 224 224 224 369 369 369 +NULL NULL 333444 555666 333444 555666 Index: ql/src/test/results/clientpositive/spark/ppd_join4.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ppd_join4.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ppd_join4.q.out (working copy) @@ -81,7 +81,7 @@ Reduce Output Operator key expressions: 'a' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 'a' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -90,7 +90,7 @@ Reduce Output Operator key expressions: 'a' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 'a' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reducer 3 Reduce Operator Tree: Index: ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out (working copy) @@ -82,9 +82,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Reducer 2 @@ -141,9 +141,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Map 3 @@ -155,9 +155,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Map 4 @@ -184,10 +184,10 @@ 0 id (type: int) 1 id (type: int) 2 id (type: int) - outputColumnNames: _col1, _col2, _col7, _col8, _col12, _col13, _col14 + outputColumnNames: _col0, _col1, _col2, _col7, _col8, _col12, _col13, _col14 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: 20 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -228,9 +228,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Map 3 @@ -242,9 +242,9 @@ predicate: (id = 20) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: 20 (type: int) + key expressions: id (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: id (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string), value (type: string) Map 4 @@ -271,10 +271,10 @@ 0 id (type: int) 1 id (type: int) 2 id (type: int) - outputColumnNames: _col1, _col2, _col7, _col8, _col12, _col13, _col14 + outputColumnNames: _col0, _col1, _col2, _col7, _col8, _col12, _col13, _col14 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: 20 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), 20 (type: int), _col7 (type: string), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator Index: ql/src/test/results/clientpositive/spark/ppd_transform.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ppd_transform.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ppd_transform.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM src SELECT TRANSFORM(src.key, src.value) @@ -7,7 +9,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM ( FROM src SELECT TRANSFORM(src.key, src.value) Index: ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out (working copy) @@ -34,7 +34,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@flights_tiny -PREHOOK: query: -- 1. basic Matchpath test +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,7 +49,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny #### A masked pattern was here #### -POSTHOOK: query: -- 1. basic Matchpath test +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -65,17 +69,17 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 Chicago 1531 2010 10 21 2 21 Chicago 1531 2010 10 25 3 25 Chicago 1531 2010 10 26 2 26 -Baltimore 1599 2010 10 21 2 21 -Baltimore 1599 2010 10 25 3 25 -Baltimore 1599 2010 10 26 2 26 Chicago 361 2010 10 20 2 20 -Washington 7291 2010 10 27 2 27 Chicago 897 2010 10 20 4 20 Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on Index: ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ptf_rcfile.q.out (working copy) @@ -38,7 +38,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@part_rc -PREHOOK: query: -- testWindowingPTFWithPartRC +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartRC select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, @@ -49,7 +51,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@part_rc #### A masked pattern was here #### -POSTHOOK: query: -- testWindowingPTFWithPartRC +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartRC select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, Index: ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ptf_register_tblfn.q.out (working copy) @@ -40,7 +40,9 @@ POSTHOOK: query: create temporary function matchpathtest as 'org.apache.hadoop.hive.ql.udf.ptf.MatchPath$MatchPathResolver' POSTHOOK: type: CREATEFUNCTION POSTHOOK: Output: matchpathtest -PREHOOK: query: -- 1. basic Matchpath test +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpathtest(on flights_tiny @@ -53,7 +55,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny #### A masked pattern was here #### -POSTHOOK: query: -- 1. basic Matchpath test +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpathtest(on flights_tiny @@ -71,17 +75,17 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 Chicago 1531 2010 10 21 2 21 Chicago 1531 2010 10 25 3 25 Chicago 1531 2010 10 26 2 26 -Baltimore 1599 2010 10 21 2 21 -Baltimore 1599 2010 10 25 3 25 -Baltimore 1599 2010 10 26 2 26 Chicago 361 2010 10 20 2 20 -Washington 7291 2010 10 27 2 27 Chicago 897 2010 10 20 4 20 Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 PREHOOK: query: drop temporary function matchpathtest PREHOOK: type: DROPFUNCTION PREHOOK: Output: matchpathtest Index: ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/ptf_seqfile.q.out (working copy) @@ -38,7 +38,9 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@part_seq -PREHOOK: query: -- testWindowingPTFWithPartSeqFile +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartSeqFile select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, @@ -49,7 +51,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@part_seq #### A masked pattern was here #### -POSTHOOK: query: -- testWindowingPTFWithPartSeqFile +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- testWindowingPTFWithPartSeqFile select p_mfgr, p_name, p_size, rank() over (partition by p_mfgr order by p_name) as r, dense_rank() over (partition by p_mfgr order by p_name) as dr, Index: ql/src/test/results/clientpositive/spark/sample3.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/sample3.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/sample3.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: -- no input pruning, sample filter +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN SELECT s.key FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s PREHOOK: type: QUERY -POSTHOOK: query: -- no input pruning, sample filter +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN SELECT s.key FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s @@ -43,48 +47,8 @@ 0 0 0 -5 -5 -5 -5 10 10 -15 -15 -15 -15 -20 -20 -20 -30 -30 -35 -35 -35 -35 -35 -35 -40 -40 -50 -60 -65 -65 -65 -70 -70 -70 -70 -75 -80 -80 -85 -85 -90 -90 -90 -95 -95 100 100 100 @@ -104,6 +68,10 @@ 135 140 145 +15 +15 +15 +15 150 155 160 @@ -122,6 +90,9 @@ 190 195 195 +20 +20 +20 200 200 205 @@ -155,6 +126,8 @@ 285 295 295 +30 +30 300 300 305 @@ -172,6 +145,12 @@ 335 340 345 +35 +35 +35 +35 +35 +35 355 355 360 @@ -192,6 +171,8 @@ 395 395 395 +40 +40 400 405 410 @@ -227,3 +208,26 @@ 490 495 495 +5 +5 +5 +5 +50 +60 +65 +65 +65 +70 +70 +70 +70 +75 +80 +80 +85 +85 +90 +90 +90 +95 +95 Index: ql/src/test/results/clientpositive/spark/sample5.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/sample5.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/sample5.q.out (working copy) @@ -6,12 +6,16 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dest1 -PREHOOK: query: -- no input pruning, sample filter +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s PREHOOK: type: QUERY -POSTHOOK: query: -- no input pruning, sample filter +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- no input pruning, sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s @@ -194,48 +198,8 @@ 0 val_0 0 val_1 0 val_1 -5 val_5 -5 val_5 -5 val_5 -5 val_6 10 val_10 10 val_11 -15 val_15 -15 val_15 -15 val_16 -15 val_16 -20 val_20 -20 val_21 -20 val_21 -30 val_30 -30 val_31 -35 val_35 -35 val_35 -35 val_35 -35 val_36 -35 val_36 -35 val_36 -40 val_41 -40 val_41 -50 val_51 -60 val_61 -65 val_65 -65 val_66 -65 val_66 -70 val_70 -70 val_70 -70 val_70 -70 val_71 -75 val_76 -80 val_80 -80 val_81 -85 val_85 -85 val_86 -90 val_90 -90 val_90 -90 val_90 -95 val_95 -95 val_95 100 val_100 100 val_100 100 val_101 @@ -255,6 +219,10 @@ 135 val_136 140 val_141 145 val_145 +15 val_15 +15 val_15 +15 val_16 +15 val_16 150 val_150 155 val_155 160 val_160 @@ -273,6 +241,9 @@ 190 val_190 195 val_195 195 val_195 +20 val_20 +20 val_21 +20 val_21 200 val_200 200 val_200 205 val_205 @@ -306,6 +277,8 @@ 285 val_286 295 val_296 295 val_296 +30 val_30 +30 val_31 300 val_301 300 val_301 305 val_305 @@ -323,6 +296,12 @@ 335 val_336 340 val_341 345 val_345 +35 val_35 +35 val_35 +35 val_35 +35 val_36 +35 val_36 +35 val_36 355 val_356 355 val_356 360 val_360 @@ -343,6 +322,8 @@ 395 val_395 395 val_396 395 val_396 +40 val_41 +40 val_41 400 val_400 405 val_406 410 val_411 @@ -378,3 +359,26 @@ 490 val_491 495 val_495 495 val_496 +5 val_5 +5 val_5 +5 val_5 +5 val_6 +50 val_51 +60 val_61 +65 val_65 +65 val_66 +65 val_66 +70 val_70 +70 val_70 +70 val_70 +70 val_71 +75 val_76 +80 val_80 +80 val_81 +85 val_85 +85 val_86 +90 val_90 +90 val_90 +90 val_90 +95 val_95 +95 val_95 Index: ql/src/test/results/clientpositive/spark/scriptfile1.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/scriptfile1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/scriptfile1.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: -- EXCLUDE_OS_WINDOWS +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: query: -- EXCLUDE_OS_WINDOWS +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default Index: ql/src/test/results/clientpositive/spark/semijoin.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/semijoin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/semijoin.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: create table t1 as select cast(key as int) key, value from src where key <= 10 +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table t1 as select cast(key as int) key, value from src where key <= 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1 as select cast(key as int) key, value from src where key <= 10 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table t1 as select cast(key as int) key, value from src where key <= 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -19,6 +23,7 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 2 val_2 4 val_4 5 val_5 @@ -26,7 +31,6 @@ 5 val_5 8 val_8 9 val_9 -10 val_10 PREHOOK: query: create table t2 as select cast(2*key as int) key, value from t1 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@t1 @@ -48,14 +52,14 @@ 0 val_0 0 val_0 0 val_0 -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 16 val_8 18 val_9 20 val_10 +4 val_2 +8 val_4 PREHOOK: query: create table t3 as select * from (select * from t1 union all select * from t2) b PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@t1 @@ -82,7 +86,14 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 +10 val_5 +10 val_5 +10 val_5 +16 val_8 +18 val_9 2 val_2 +20 val_10 4 val_2 4 val_4 5 val_5 @@ -91,13 +102,6 @@ 8 val_4 8 val_8 9 val_9 -10 val_10 -10 val_5 -10 val_5 -10 val_5 -16 val_8 -18 val_9 -20 val_10 PREHOOK: query: create table t4 (key int, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -213,9 +217,9 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 4 val_4 8 val_8 -10 val_10 PREHOOK: query: explain select * from t2 a left semi join t1 b on b.key=a.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t2 a left semi join t1 b on b.key=a.key sort by a.key, a.value @@ -315,11 +319,11 @@ 0 val_0 0 val_0 0 val_0 -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 +4 val_2 +8 val_4 PREHOOK: query: explain select * from t1 a left semi join t4 b on b.key=a.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 a left semi join t4 b on b.key=a.key sort by a.key, a.value @@ -921,11 +925,11 @@ POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### -4 val_2 -8 val_4 10 val_5 10 val_5 10 val_5 +4 val_2 +8 val_4 PREHOOK: query: explain select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key @@ -1030,6 +1034,10 @@ 0 0 0 +10 +10 +10 +10 2 4 4 @@ -1039,10 +1047,6 @@ 8 8 9 -10 -10 -10 -10 PREHOOK: query: explain select * from t1 a left semi join t2 b on a.key = 2*b.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 a left semi join t2 b on a.key = 2*b.key sort by a.key, a.value @@ -1271,11 +1275,11 @@ 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 -4 val_4 4 val_2 -8 val_8 8 val_4 10 val_10 10 val_5 10 val_10 10 val_5 10 val_10 10 val_5 +4 val_4 4 val_2 +8 val_8 8 val_4 PREHOOK: query: explain select * from t3 a left semi join t1 b on a.key = b.key and a.value=b.value sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t3 a left semi join t1 b on a.key = b.key and a.value=b.value sort by a.key, a.value @@ -1377,6 +1381,7 @@ 0 val_0 0 val_0 0 val_0 +10 val_10 2 val_2 4 val_4 5 val_5 @@ -1384,7 +1389,6 @@ 5 val_5 8 val_8 9 val_9 -10 val_10 PREHOOK: query: explain select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key @@ -1519,14 +1523,14 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 PREHOOK: query: explain select a.key from t3 a left outer join t1 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left outer join t1 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1648,14 +1652,14 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 PREHOOK: query: explain select a.key from t1 a right outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t1 a right outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1759,9 +1763,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -1780,14 +1781,17 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t1 a full outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t1 a full outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key @@ -1891,9 +1895,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -1912,14 +1913,17 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.key = c.key sort by a.key @@ -2041,10 +2045,6 @@ 0 0 0 -4 -4 -8 -8 10 10 10 @@ -2052,6 +2052,10 @@ 16 18 20 +4 +4 +8 +8 PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key right outer join t1 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key right outer join t1 c on a.key = c.key sort by a.key @@ -2155,11 +2159,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL 0 0 0 @@ -2178,14 +2177,19 @@ 0 0 0 +10 +10 +10 +10 4 4 8 8 -10 -10 -10 -10 +NULL +NULL +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t1 b on a.key = b.key full outer join t2 c on a.key = c.key sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t1 b on a.key = b.key full outer join t2 c on a.key = c.key sort by a.key @@ -2289,9 +2293,6 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### -NULL -NULL -NULL 0 0 0 @@ -2310,15 +2311,6 @@ 0 0 0 -2 -4 -4 -5 -5 -5 -8 -8 -9 10 10 10 @@ -2331,6 +2323,18 @@ 10 10 10 +2 +4 +4 +5 +5 +5 +8 +8 +9 +NULL +NULL +NULL PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key @@ -2474,10 +2478,6 @@ 0 0 0 -4 -4 -8 -8 10 10 10 @@ -2491,6 +2491,10 @@ 16 18 20 +4 +4 +8 +8 PREHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value where a.key > 100 PREHOOK: type: QUERY POSTHOOK: query: explain select a.key from t3 a left semi join t2 b on a.value = b.value where a.key > 100 Index: ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out (working copy) @@ -283,6 +283,1895 @@ POSTHOOK: Output: default@test_table3@ds=1 POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value SIMPLE [(test_table2)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM test_table1 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: SELECT * FROM test_table3 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +5 val_5 1 +8 val_8 1 +9 val_9 1 +10 val_10 1 +11 val_11 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +12 val_12 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +15 val_15 1 +17 val_17 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +18 val_18 1 +19 val_19 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +26 val_26 1 +27 val_27 1 +28 val_28 1 +30 val_30 1 +33 val_33 1 +34 val_34 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +35 val_35 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +37 val_37 1 +41 val_41 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +42 val_42 1 +43 val_43 1 +44 val_44 1 +47 val_47 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +83 val_83 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +100 val_100 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +103 val_103 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +104 val_104 1 +105 val_105 1 +111 val_111 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +118 val_118 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +119 val_119 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +120 val_120 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +137 val_137 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +146 val_146 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +149 val_149 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +164 val_164 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +169 val_169 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +174 val_174 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +175 val_175 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +179 val_179 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +187 val_187 1 +189 val_189 1 +190 val_190 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +197 val_197 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +199 val_199 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +203 val_203 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +205 val_205 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +207 val_207 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +216 val_216 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +219 val_219 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +223 val_223 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +229 val_229 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +237 val_237 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +238 val_238 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +239 val_239 1 +241 val_241 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +255 val_255 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +265 val_265 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +272 val_272 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +277 val_277 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +280 val_280 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +281 val_281 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +317 val_317 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +325 val_325 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +327 val_327 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +341 val_341 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +351 val_351 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +395 val_395 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +397 val_397 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +399 val_399 1 +400 val_400 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +403 val_403 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +409 val_409 1 +411 val_411 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +413 val_413 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +414 val_414 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +421 val_421 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +429 val_429 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +439 val_439 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +458 val_458 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +462 val_462 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +463 val_463 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +469 val_469 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_table1 + TOK_TABLEBUCKETSAMPLE + 2 + 16 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + numFiles 16 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table1 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table1 + name: default.test_table1 + Processor Tree: + TableScan + alias: test_table1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_table3 + TOK_TABLEBUCKETSAMPLE + 2 + 16 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + numFiles 16 + numRows 1028 + partition_columns ds + partition_columns.types string + rawDataSize 10968 + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11996 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + name: default.test_table3 + Processor Tree: + TableScan + alias: test_table3 + Statistics: Num rows: 1028 Data size: 10968 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table1 +PREHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table1 +POSTHOOK: Input: default@test_table1@ds=1 +#### A masked pattern was here #### +17 val_17 1 +33 val_33 1 +65 val_65 1 +97 val_97 1 +97 val_97 1 +113 val_113 1 +113 val_113 1 +129 val_129 1 +129 val_129 1 +145 val_145 1 +177 val_177 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +209 val_209 1 +209 val_209 1 +241 val_241 1 +257 val_257 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +289 val_289 1 +305 val_305 1 +321 val_321 1 +321 val_321 1 +353 val_353 1 +353 val_353 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +449 val_449 1 +481 val_481 1 +497 val_497 1 +PREHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table3 +PREHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table3 +POSTHOOK: Input: default@test_table3@ds=1 +#### A masked pattern was here #### +17 val_17 1 +33 val_33 1 +65 val_65 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +97 val_97 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +113 val_113 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +129 val_129 1 +145 val_145 1 +177 val_177 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +193 val_193 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +209 val_209 1 +241 val_241 1 +257 val_257 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +273 val_273 1 +289 val_289 1 +305 val_305 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +321 val_321 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +353 val_353 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +369 val_369 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +401 val_401 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +417 val_417 1 +449 val_449 1 +481 val_481 1 +497 val_497 1 PREHOOK: query: -- Join data from a sampled bucket to verify the data is bucketed SELECT COUNT(*) FROM test_table3 TABLESAMPLE(BUCKET 2 OUT OF 16) a JOIN test_table1 TABLESAMPLE(BUCKET 2 OUT OF 16) b ON a.key = b.key AND a.ds = '1' AND b.ds='1' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out (working copy) @@ -76,7 +76,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -89,7 +89,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: @@ -102,7 +102,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: @@ -115,7 +115,7 @@ Reduce Output Operator key expressions: 5 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 5 (type: int) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Index: ql/src/test/results/clientpositive/spark/sort.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/sort.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/sort.q.out (working copy) @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT x.* FROM SRC x SORT BY key PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN SELECT x.* FROM SRC x SORT BY key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Index: ql/src/test/results/clientpositive/spark/stats1.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/stats1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/stats1.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable Index: ql/src/test/results/clientpositive/spark/subquery_in.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/subquery_in.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/subquery_in.q.out (working copy) @@ -140,7 +140,7 @@ Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -265,9 +265,9 @@ Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2) Reducer 5 <- Reducer 4 (GROUP, 1) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -294,7 +294,6 @@ sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: Join Operator @@ -318,7 +317,9 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -413,9 +414,9 @@ Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3) - Reducer 5 <- Reducer 4 (GROUP, 3) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2) + Reducer 5 <- Reducer 4 (GROUP, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -442,7 +443,6 @@ sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: Join Operator @@ -466,7 +466,9 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -573,8 +575,8 @@ Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 3) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 4 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 3 (GROUP, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -728,9 +730,9 @@ Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) - Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Map 6 (PARTITION-LEVEL SORT, 3), Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 Index: ql/src/test/results/clientpositive/spark/transform_ppr1.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/transform_ppr1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/transform_ppr1.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -7,7 +9,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/spark/transform_ppr2.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/transform_ppr2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/transform_ppr2.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -8,7 +10,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/spark/union10.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/union10.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/union10.q.out (working copy) @@ -1,11 +1,15 @@ -PREHOOK: query: -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + create table tmptable(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + create table tmptable(key string, value int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default Index: ql/src/test/results/clientpositive/spark/union18.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/union18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/union18.q.out (working copy) @@ -14,8 +14,10 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST2 -PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case:map-reduce sub-queries followed by multi-table insert + explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL @@ -23,8 +25,10 @@ INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY -POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case:map-reduce sub-queries followed by multi-table insert + explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL Index: ql/src/test/results/clientpositive/spark/union19.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/union19.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/union19.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST1 -POSTHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST1 Index: ql/src/test/results/clientpositive/spark/union6.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/union6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/union6.q.out (working copy) @@ -1,12 +1,14 @@ -PREHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE Index: ql/src/test/results/clientpositive/spark/union_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/union_ppr.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/union_ppr.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED SELECT * FROM ( SELECT X.* FROM SRCPART X WHERE X.key < 100 UNION ALL @@ -7,7 +9,9 @@ WHERE A.ds = '2008-04-08' SORT BY A.key, A.value, A.ds, A.hr PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED SELECT * FROM ( SELECT X.* FROM SRCPART X WHERE X.key < 100 UNION ALL Index: ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out (working copy) @@ -223,7 +223,7 @@ Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col1 (type: int), 1 (type: int) + 0 _col1 (type: int), _col4 (type: int) 1 _col0 (type: int), _col1 (type: int) Local Work: Map Reduce Local Work @@ -267,7 +267,7 @@ keys: 0 _col0 (type: int) 1 l_partkey (type: int) - outputColumnNames: _col0, _col1, _col3 + outputColumnNames: _col0, _col1, _col3, _col4 input vertices: 1 Map 3 Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE @@ -275,7 +275,7 @@ condition map: Left Semi Join 0 to 1 keys: - 0 _col1 (type: int), 1 (type: int) + 0 _col1 (type: int), _col4 (type: int) 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col3 input vertices: Index: ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out =================================================================== --- ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out (working copy) @@ -312,12 +312,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -689,12 +691,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -1171,12 +1175,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1459,12 +1465,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1777,13 +1785,15 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col2 (type: int) auto parallelism: false Execution mode: vectorized Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -2658,12 +2668,13 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2935,12 +2946,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3214,12 +3227,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3520,12 +3535,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3829,12 +3846,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4305,12 +4324,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -4838,12 +4859,14 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col2 (type: double) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -5270,7 +5293,9 @@ Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5280,12 +5305,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _wcol0 (type: bigint), _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _wcol0 (type: bigint), _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: bigint), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5322,7 +5349,9 @@ Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5369,7 +5398,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string) auto parallelism: false Reducer 7 Needs Tagging: false @@ -5388,7 +5417,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Stage: Stage-1 @@ -5813,12 +5842,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6183,12 +6214,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6521,12 +6554,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6888,12 +6923,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7271,12 +7308,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7620,12 +7659,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/stats1.q.out =================================================================== --- ql/src/test/results/clientpositive/stats1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/stats1.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable Index: ql/src/test/results/clientpositive/subquery_in.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_in.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/subquery_in.q.out (working copy) @@ -271,9 +271,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -439,9 +440,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/subquery_in_having.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_in_having.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/subquery_in_having.q.out (working copy) @@ -1379,9 +1379,11 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/subquery_notin.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_notin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/subquery_notin.q.out (working copy) @@ -330,9 +330,11 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -468,9 +470,11 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -579,9 +583,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -714,9 +719,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -873,9 +879,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1048,9 +1055,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out (working copy) @@ -224,9 +224,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -376,9 +377,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -810,9 +812,11 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -948,9 +952,11 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col0 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/tez/auto_join21.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join21.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/auto_join21.q.out (working copy) @@ -0,0 +1,610 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + filter predicates: + 0 {(KEY.reducesinkkey0 < 10)} + 1 + 2 {(KEY.reducesinkkey0 < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 105 val_105 +NULL NULL NULL NULL 11 val_11 +NULL NULL NULL NULL 111 val_111 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 114 val_114 +NULL NULL NULL NULL 116 val_116 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 126 val_126 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 131 val_131 +NULL NULL NULL NULL 133 val_133 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 136 val_136 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 143 val_143 +NULL NULL NULL NULL 145 val_145 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 150 val_150 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 153 val_153 +NULL NULL NULL NULL 155 val_155 +NULL NULL NULL NULL 156 val_156 +NULL NULL NULL NULL 157 val_157 +NULL NULL NULL NULL 158 val_158 +NULL NULL NULL NULL 160 val_160 +NULL NULL NULL NULL 162 val_162 +NULL NULL NULL NULL 163 val_163 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 166 val_166 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 168 val_168 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 170 val_170 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 177 val_177 +NULL NULL NULL NULL 178 val_178 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 180 val_180 +NULL NULL NULL NULL 181 val_181 +NULL NULL NULL NULL 183 val_183 +NULL NULL NULL NULL 186 val_186 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 189 val_189 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 190 val_190 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 192 val_192 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 194 val_194 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 196 val_196 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 2 val_2 +NULL NULL NULL NULL 20 val_20 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 201 val_201 +NULL NULL NULL NULL 202 val_202 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 214 val_214 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 218 val_218 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 222 val_222 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 226 val_226 +NULL NULL NULL NULL 228 val_228 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 235 val_235 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 241 val_241 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 244 val_244 +NULL NULL NULL NULL 247 val_247 +NULL NULL NULL NULL 248 val_248 +NULL NULL NULL NULL 249 val_249 +NULL NULL NULL NULL 252 val_252 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 257 val_257 +NULL NULL NULL NULL 258 val_258 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 260 val_260 +NULL NULL NULL NULL 262 val_262 +NULL NULL NULL NULL 263 val_263 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 266 val_266 +NULL NULL NULL NULL 27 val_27 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 274 val_274 +NULL NULL NULL NULL 275 val_275 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 28 val_28 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 283 val_283 +NULL NULL NULL NULL 284 val_284 +NULL NULL NULL NULL 285 val_285 +NULL NULL NULL NULL 286 val_286 +NULL NULL NULL NULL 287 val_287 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 289 val_289 +NULL NULL NULL NULL 291 val_291 +NULL NULL NULL NULL 292 val_292 +NULL NULL NULL NULL 296 val_296 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 30 val_30 +NULL NULL NULL NULL 302 val_302 +NULL NULL NULL NULL 305 val_305 +NULL NULL NULL NULL 306 val_306 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 308 val_308 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 310 val_310 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 315 val_315 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 323 val_323 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 33 val_33 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 332 val_332 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 335 val_335 +NULL NULL NULL NULL 336 val_336 +NULL NULL NULL NULL 338 val_338 +NULL NULL NULL NULL 339 val_339 +NULL NULL NULL NULL 34 val_34 +NULL NULL NULL NULL 341 val_341 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 345 val_345 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 351 val_351 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 356 val_356 +NULL NULL NULL NULL 360 val_360 +NULL NULL NULL NULL 362 val_362 +NULL NULL NULL NULL 364 val_364 +NULL NULL NULL NULL 365 val_365 +NULL NULL NULL NULL 366 val_366 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 368 val_368 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 373 val_373 +NULL NULL NULL NULL 374 val_374 +NULL NULL NULL NULL 375 val_375 +NULL NULL NULL NULL 377 val_377 +NULL NULL NULL NULL 378 val_378 +NULL NULL NULL NULL 379 val_379 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 386 val_386 +NULL NULL NULL NULL 389 val_389 +NULL NULL NULL NULL 392 val_392 +NULL NULL NULL NULL 393 val_393 +NULL NULL NULL NULL 394 val_394 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 4 val_4 +NULL NULL NULL NULL 400 val_400 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 402 val_402 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 407 val_407 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 41 val_41 +NULL NULL NULL NULL 411 val_411 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 418 val_418 +NULL NULL NULL NULL 419 val_419 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 421 val_421 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 427 val_427 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 43 val_43 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 432 val_432 +NULL NULL NULL NULL 435 val_435 +NULL NULL NULL NULL 436 val_436 +NULL NULL NULL NULL 437 val_437 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 44 val_44 +NULL NULL NULL NULL 443 val_443 +NULL NULL NULL NULL 444 val_444 +NULL NULL NULL NULL 446 val_446 +NULL NULL NULL NULL 448 val_448 +NULL NULL NULL NULL 449 val_449 +NULL NULL NULL NULL 452 val_452 +NULL NULL NULL NULL 453 val_453 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 455 val_455 +NULL NULL NULL NULL 457 val_457 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 460 val_460 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 467 val_467 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 47 val_47 +NULL NULL NULL NULL 470 val_470 +NULL NULL NULL NULL 472 val_472 +NULL NULL NULL NULL 475 val_475 +NULL NULL NULL NULL 477 val_477 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 479 val_479 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 481 val_481 +NULL NULL NULL NULL 482 val_482 +NULL NULL NULL NULL 483 val_483 +NULL NULL NULL NULL 484 val_484 +NULL NULL NULL NULL 485 val_485 +NULL NULL NULL NULL 487 val_487 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 490 val_490 +NULL NULL NULL NULL 491 val_491 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 493 val_493 +NULL NULL NULL NULL 494 val_494 +NULL NULL NULL NULL 495 val_495 +NULL NULL NULL NULL 496 val_496 +NULL NULL NULL NULL 497 val_497 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 53 val_53 +NULL NULL NULL NULL 54 val_54 +NULL NULL NULL NULL 57 val_57 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 64 val_64 +NULL NULL NULL NULL 65 val_65 +NULL NULL NULL NULL 66 val_66 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 69 val_69 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 74 val_74 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 77 val_77 +NULL NULL NULL NULL 78 val_78 +NULL NULL NULL NULL 8 val_8 +NULL NULL NULL NULL 80 val_80 +NULL NULL NULL NULL 82 val_82 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 85 val_85 +NULL NULL NULL NULL 86 val_86 +NULL NULL NULL NULL 87 val_87 +NULL NULL NULL NULL 9 val_9 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 92 val_92 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 96 val_96 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 98 val_98 +NULL NULL NULL NULL 98 val_98 Index: ql/src/test/results/clientpositive/tez/auto_join29.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join29.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/auto_join29.q.out (working copy) @@ -0,0 +1,3011 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + filter predicates: + 0 {(KEY.reducesinkkey0 < 10)} + 1 + 2 {(KEY.reducesinkkey0 < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 0 val_0 +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 105 val_105 +NULL NULL NULL NULL 11 val_11 +NULL NULL NULL NULL 111 val_111 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 114 val_114 +NULL NULL NULL NULL 116 val_116 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 126 val_126 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 131 val_131 +NULL NULL NULL NULL 133 val_133 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 136 val_136 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 143 val_143 +NULL NULL NULL NULL 145 val_145 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 150 val_150 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 153 val_153 +NULL NULL NULL NULL 155 val_155 +NULL NULL NULL NULL 156 val_156 +NULL NULL NULL NULL 157 val_157 +NULL NULL NULL NULL 158 val_158 +NULL NULL NULL NULL 160 val_160 +NULL NULL NULL NULL 162 val_162 +NULL NULL NULL NULL 163 val_163 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 166 val_166 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 168 val_168 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 170 val_170 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 177 val_177 +NULL NULL NULL NULL 178 val_178 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 180 val_180 +NULL NULL NULL NULL 181 val_181 +NULL NULL NULL NULL 183 val_183 +NULL NULL NULL NULL 186 val_186 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 189 val_189 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 190 val_190 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 192 val_192 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 194 val_194 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 196 val_196 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 2 val_2 +NULL NULL NULL NULL 20 val_20 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 201 val_201 +NULL NULL NULL NULL 202 val_202 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 214 val_214 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 218 val_218 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 222 val_222 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 226 val_226 +NULL NULL NULL NULL 228 val_228 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 235 val_235 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 241 val_241 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 244 val_244 +NULL NULL NULL NULL 247 val_247 +NULL NULL NULL NULL 248 val_248 +NULL NULL NULL NULL 249 val_249 +NULL NULL NULL NULL 252 val_252 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 257 val_257 +NULL NULL NULL NULL 258 val_258 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 260 val_260 +NULL NULL NULL NULL 262 val_262 +NULL NULL NULL NULL 263 val_263 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 266 val_266 +NULL NULL NULL NULL 27 val_27 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 274 val_274 +NULL NULL NULL NULL 275 val_275 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 28 val_28 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 283 val_283 +NULL NULL NULL NULL 284 val_284 +NULL NULL NULL NULL 285 val_285 +NULL NULL NULL NULL 286 val_286 +NULL NULL NULL NULL 287 val_287 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 289 val_289 +NULL NULL NULL NULL 291 val_291 +NULL NULL NULL NULL 292 val_292 +NULL NULL NULL NULL 296 val_296 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 30 val_30 +NULL NULL NULL NULL 302 val_302 +NULL NULL NULL NULL 305 val_305 +NULL NULL NULL NULL 306 val_306 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 308 val_308 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 310 val_310 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 315 val_315 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 323 val_323 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 33 val_33 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 332 val_332 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 335 val_335 +NULL NULL NULL NULL 336 val_336 +NULL NULL NULL NULL 338 val_338 +NULL NULL NULL NULL 339 val_339 +NULL NULL NULL NULL 34 val_34 +NULL NULL NULL NULL 341 val_341 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 345 val_345 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 351 val_351 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 356 val_356 +NULL NULL NULL NULL 360 val_360 +NULL NULL NULL NULL 362 val_362 +NULL NULL NULL NULL 364 val_364 +NULL NULL NULL NULL 365 val_365 +NULL NULL NULL NULL 366 val_366 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 368 val_368 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 373 val_373 +NULL NULL NULL NULL 374 val_374 +NULL NULL NULL NULL 375 val_375 +NULL NULL NULL NULL 377 val_377 +NULL NULL NULL NULL 378 val_378 +NULL NULL NULL NULL 379 val_379 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 386 val_386 +NULL NULL NULL NULL 389 val_389 +NULL NULL NULL NULL 392 val_392 +NULL NULL NULL NULL 393 val_393 +NULL NULL NULL NULL 394 val_394 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 4 val_4 +NULL NULL NULL NULL 400 val_400 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 402 val_402 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 407 val_407 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 41 val_41 +NULL NULL NULL NULL 411 val_411 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 418 val_418 +NULL NULL NULL NULL 419 val_419 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 421 val_421 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 427 val_427 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 43 val_43 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 432 val_432 +NULL NULL NULL NULL 435 val_435 +NULL NULL NULL NULL 436 val_436 +NULL NULL NULL NULL 437 val_437 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 44 val_44 +NULL NULL NULL NULL 443 val_443 +NULL NULL NULL NULL 444 val_444 +NULL NULL NULL NULL 446 val_446 +NULL NULL NULL NULL 448 val_448 +NULL NULL NULL NULL 449 val_449 +NULL NULL NULL NULL 452 val_452 +NULL NULL NULL NULL 453 val_453 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 455 val_455 +NULL NULL NULL NULL 457 val_457 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 460 val_460 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 467 val_467 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 47 val_47 +NULL NULL NULL NULL 470 val_470 +NULL NULL NULL NULL 472 val_472 +NULL NULL NULL NULL 475 val_475 +NULL NULL NULL NULL 477 val_477 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 479 val_479 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 481 val_481 +NULL NULL NULL NULL 482 val_482 +NULL NULL NULL NULL 483 val_483 +NULL NULL NULL NULL 484 val_484 +NULL NULL NULL NULL 485 val_485 +NULL NULL NULL NULL 487 val_487 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 490 val_490 +NULL NULL NULL NULL 491 val_491 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 493 val_493 +NULL NULL NULL NULL 494 val_494 +NULL NULL NULL NULL 495 val_495 +NULL NULL NULL NULL 496 val_496 +NULL NULL NULL NULL 497 val_497 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 5 val_5 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 53 val_53 +NULL NULL NULL NULL 54 val_54 +NULL NULL NULL NULL 57 val_57 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 64 val_64 +NULL NULL NULL NULL 65 val_65 +NULL NULL NULL NULL 66 val_66 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 69 val_69 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 74 val_74 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 77 val_77 +NULL NULL NULL NULL 78 val_78 +NULL NULL NULL NULL 8 val_8 +NULL NULL NULL NULL 80 val_80 +NULL NULL NULL NULL 82 val_82 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 85 val_85 +NULL NULL NULL NULL 86 val_86 +NULL NULL NULL NULL 87 val_87 +NULL NULL NULL NULL 9 val_9 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 92 val_92 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 96 val_96 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 98 val_98 +NULL NULL NULL NULL 98 val_98 +PREHOOK: query: explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + filter predicates: + 0 {(KEY.reducesinkkey0 < 10)} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 NULL NULL NULL NULL +0 val_0 NULL NULL NULL NULL +0 val_0 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL +100 val_100 NULL NULL NULL NULL +100 val_100 NULL NULL NULL NULL +103 val_103 NULL NULL NULL NULL +103 val_103 NULL NULL NULL NULL +104 val_104 NULL NULL NULL NULL +104 val_104 NULL NULL NULL NULL +105 val_105 NULL NULL NULL NULL +11 val_11 NULL NULL NULL NULL +111 val_111 NULL NULL NULL NULL +113 val_113 NULL NULL NULL NULL +113 val_113 NULL NULL NULL NULL +114 val_114 NULL NULL NULL NULL +116 val_116 NULL NULL NULL NULL +118 val_118 NULL NULL NULL NULL +118 val_118 NULL NULL NULL NULL +119 val_119 NULL NULL NULL NULL +119 val_119 NULL NULL NULL NULL +119 val_119 NULL NULL NULL NULL +12 val_12 NULL NULL NULL NULL +12 val_12 NULL NULL NULL NULL +120 val_120 NULL NULL NULL NULL +120 val_120 NULL NULL NULL NULL +125 val_125 NULL NULL NULL NULL +125 val_125 NULL NULL NULL NULL +126 val_126 NULL NULL NULL NULL +128 val_128 NULL NULL NULL NULL +128 val_128 NULL NULL NULL NULL +128 val_128 NULL NULL NULL NULL +129 val_129 NULL NULL NULL NULL +129 val_129 NULL NULL NULL NULL +131 val_131 NULL NULL NULL NULL +133 val_133 NULL NULL NULL NULL +134 val_134 NULL NULL NULL NULL +134 val_134 NULL NULL NULL NULL +136 val_136 NULL NULL NULL NULL +137 val_137 NULL NULL NULL NULL +137 val_137 NULL NULL NULL NULL +138 val_138 NULL NULL NULL NULL +138 val_138 NULL NULL NULL NULL +138 val_138 NULL NULL NULL NULL +138 val_138 NULL NULL NULL NULL +143 val_143 NULL NULL NULL NULL +145 val_145 NULL NULL NULL NULL +146 val_146 NULL NULL NULL NULL +146 val_146 NULL NULL NULL NULL +149 val_149 NULL NULL NULL NULL +149 val_149 NULL NULL NULL NULL +15 val_15 NULL NULL NULL NULL +15 val_15 NULL NULL NULL NULL +150 val_150 NULL NULL NULL NULL +152 val_152 NULL NULL NULL NULL +152 val_152 NULL NULL NULL NULL +153 val_153 NULL NULL NULL NULL +155 val_155 NULL NULL NULL NULL +156 val_156 NULL NULL NULL NULL +157 val_157 NULL NULL NULL NULL +158 val_158 NULL NULL NULL NULL +160 val_160 NULL NULL NULL NULL +162 val_162 NULL NULL NULL NULL +163 val_163 NULL NULL NULL NULL +164 val_164 NULL NULL NULL NULL +164 val_164 NULL NULL NULL NULL +165 val_165 NULL NULL NULL NULL +165 val_165 NULL NULL NULL NULL +166 val_166 NULL NULL NULL NULL +167 val_167 NULL NULL NULL NULL +167 val_167 NULL NULL NULL NULL +167 val_167 NULL NULL NULL NULL +168 val_168 NULL NULL NULL NULL +169 val_169 NULL NULL NULL NULL +169 val_169 NULL NULL NULL NULL +169 val_169 NULL NULL NULL NULL +169 val_169 NULL NULL NULL NULL +17 val_17 NULL NULL NULL NULL +170 val_170 NULL NULL NULL NULL +172 val_172 NULL NULL NULL NULL +172 val_172 NULL NULL NULL NULL +174 val_174 NULL NULL NULL NULL +174 val_174 NULL NULL NULL NULL +175 val_175 NULL NULL NULL NULL +175 val_175 NULL NULL NULL NULL +176 val_176 NULL NULL NULL NULL +176 val_176 NULL NULL NULL NULL +177 val_177 NULL NULL NULL NULL +178 val_178 NULL NULL NULL NULL +179 val_179 NULL NULL NULL NULL +179 val_179 NULL NULL NULL NULL +18 val_18 NULL NULL NULL NULL +18 val_18 NULL NULL NULL NULL +180 val_180 NULL NULL NULL NULL +181 val_181 NULL NULL NULL NULL +183 val_183 NULL NULL NULL NULL +186 val_186 NULL NULL NULL NULL +187 val_187 NULL NULL NULL NULL +187 val_187 NULL NULL NULL NULL +187 val_187 NULL NULL NULL NULL +189 val_189 NULL NULL NULL NULL +19 val_19 NULL NULL NULL NULL +190 val_190 NULL NULL NULL NULL +191 val_191 NULL NULL NULL NULL +191 val_191 NULL NULL NULL NULL +192 val_192 NULL NULL NULL NULL +193 val_193 NULL NULL NULL NULL +193 val_193 NULL NULL NULL NULL +193 val_193 NULL NULL NULL NULL +194 val_194 NULL NULL NULL NULL +195 val_195 NULL NULL NULL NULL +195 val_195 NULL NULL NULL NULL +196 val_196 NULL NULL NULL NULL +197 val_197 NULL NULL NULL NULL +197 val_197 NULL NULL NULL NULL +199 val_199 NULL NULL NULL NULL +199 val_199 NULL NULL NULL NULL +199 val_199 NULL NULL NULL NULL +2 val_2 NULL NULL NULL NULL +20 val_20 NULL NULL NULL NULL +200 val_200 NULL NULL NULL NULL +200 val_200 NULL NULL NULL NULL +201 val_201 NULL NULL NULL NULL +202 val_202 NULL NULL NULL NULL +203 val_203 NULL NULL NULL NULL +203 val_203 NULL NULL NULL NULL +205 val_205 NULL NULL NULL NULL +205 val_205 NULL NULL NULL NULL +207 val_207 NULL NULL NULL NULL +207 val_207 NULL NULL NULL NULL +208 val_208 NULL NULL NULL NULL +208 val_208 NULL NULL NULL NULL +208 val_208 NULL NULL NULL NULL +209 val_209 NULL NULL NULL NULL +209 val_209 NULL NULL NULL NULL +213 val_213 NULL NULL NULL NULL +213 val_213 NULL NULL NULL NULL +214 val_214 NULL NULL NULL NULL +216 val_216 NULL NULL NULL NULL +216 val_216 NULL NULL NULL NULL +217 val_217 NULL NULL NULL NULL +217 val_217 NULL NULL NULL NULL +218 val_218 NULL NULL NULL NULL +219 val_219 NULL NULL NULL NULL +219 val_219 NULL NULL NULL NULL +221 val_221 NULL NULL NULL NULL +221 val_221 NULL NULL NULL NULL +222 val_222 NULL NULL NULL NULL +223 val_223 NULL NULL NULL NULL +223 val_223 NULL NULL NULL NULL +224 val_224 NULL NULL NULL NULL +224 val_224 NULL NULL NULL NULL +226 val_226 NULL NULL NULL NULL +228 val_228 NULL NULL NULL NULL +229 val_229 NULL NULL NULL NULL +229 val_229 NULL NULL NULL NULL +230 val_230 NULL NULL NULL NULL +230 val_230 NULL NULL NULL NULL +230 val_230 NULL NULL NULL NULL +230 val_230 NULL NULL NULL NULL +230 val_230 NULL NULL NULL NULL +233 val_233 NULL NULL NULL NULL +233 val_233 NULL NULL NULL NULL +235 val_235 NULL NULL NULL NULL +237 val_237 NULL NULL NULL NULL +237 val_237 NULL NULL NULL NULL +238 val_238 NULL NULL NULL NULL +238 val_238 NULL NULL NULL NULL +239 val_239 NULL NULL NULL NULL +239 val_239 NULL NULL NULL NULL +24 val_24 NULL NULL NULL NULL +24 val_24 NULL NULL NULL NULL +241 val_241 NULL NULL NULL NULL +242 val_242 NULL NULL NULL NULL +242 val_242 NULL NULL NULL NULL +244 val_244 NULL NULL NULL NULL +247 val_247 NULL NULL NULL NULL +248 val_248 NULL NULL NULL NULL +249 val_249 NULL NULL NULL NULL +252 val_252 NULL NULL NULL NULL +255 val_255 NULL NULL NULL NULL +255 val_255 NULL NULL NULL NULL +256 val_256 NULL NULL NULL NULL +256 val_256 NULL NULL NULL NULL +257 val_257 NULL NULL NULL NULL +258 val_258 NULL NULL NULL NULL +26 val_26 NULL NULL NULL NULL +26 val_26 NULL NULL NULL NULL +260 val_260 NULL NULL NULL NULL +262 val_262 NULL NULL NULL NULL +263 val_263 NULL NULL NULL NULL +265 val_265 NULL NULL NULL NULL +265 val_265 NULL NULL NULL NULL +266 val_266 NULL NULL NULL NULL +27 val_27 NULL NULL NULL NULL +272 val_272 NULL NULL NULL NULL +272 val_272 NULL NULL NULL NULL +273 val_273 NULL NULL NULL NULL +273 val_273 NULL NULL NULL NULL +273 val_273 NULL NULL NULL NULL +274 val_274 NULL NULL NULL NULL +275 val_275 NULL NULL NULL NULL +277 val_277 NULL NULL NULL NULL +277 val_277 NULL NULL NULL NULL +277 val_277 NULL NULL NULL NULL +277 val_277 NULL NULL NULL NULL +278 val_278 NULL NULL NULL NULL +278 val_278 NULL NULL NULL NULL +28 val_28 NULL NULL NULL NULL +280 val_280 NULL NULL NULL NULL +280 val_280 NULL NULL NULL NULL +281 val_281 NULL NULL NULL NULL +281 val_281 NULL NULL NULL NULL +282 val_282 NULL NULL NULL NULL +282 val_282 NULL NULL NULL NULL +283 val_283 NULL NULL NULL NULL +284 val_284 NULL NULL NULL NULL +285 val_285 NULL NULL NULL NULL +286 val_286 NULL NULL NULL NULL +287 val_287 NULL NULL NULL NULL +288 val_288 NULL NULL NULL NULL +288 val_288 NULL NULL NULL NULL +289 val_289 NULL NULL NULL NULL +291 val_291 NULL NULL NULL NULL +292 val_292 NULL NULL NULL NULL +296 val_296 NULL NULL NULL NULL +298 val_298 NULL NULL NULL NULL +298 val_298 NULL NULL NULL NULL +298 val_298 NULL NULL NULL NULL +30 val_30 NULL NULL NULL NULL +302 val_302 NULL NULL NULL NULL +305 val_305 NULL NULL NULL NULL +306 val_306 NULL NULL NULL NULL +307 val_307 NULL NULL NULL NULL +307 val_307 NULL NULL NULL NULL +308 val_308 NULL NULL NULL NULL +309 val_309 NULL NULL NULL NULL +309 val_309 NULL NULL NULL NULL +310 val_310 NULL NULL NULL NULL +311 val_311 NULL NULL NULL NULL +311 val_311 NULL NULL NULL NULL +311 val_311 NULL NULL NULL NULL +315 val_315 NULL NULL NULL NULL +316 val_316 NULL NULL NULL NULL +316 val_316 NULL NULL NULL NULL +316 val_316 NULL NULL NULL NULL +317 val_317 NULL NULL NULL NULL +317 val_317 NULL NULL NULL NULL +318 val_318 NULL NULL NULL NULL +318 val_318 NULL NULL NULL NULL +318 val_318 NULL NULL NULL NULL +321 val_321 NULL NULL NULL NULL +321 val_321 NULL NULL NULL NULL +322 val_322 NULL NULL NULL NULL +322 val_322 NULL NULL NULL NULL +323 val_323 NULL NULL NULL NULL +325 val_325 NULL NULL NULL NULL +325 val_325 NULL NULL NULL NULL +327 val_327 NULL NULL NULL NULL +327 val_327 NULL NULL NULL NULL +327 val_327 NULL NULL NULL NULL +33 val_33 NULL NULL NULL NULL +331 val_331 NULL NULL NULL NULL +331 val_331 NULL NULL NULL NULL +332 val_332 NULL NULL NULL NULL +333 val_333 NULL NULL NULL NULL +333 val_333 NULL NULL NULL NULL +335 val_335 NULL NULL NULL NULL +336 val_336 NULL NULL NULL NULL +338 val_338 NULL NULL NULL NULL +339 val_339 NULL NULL NULL NULL +34 val_34 NULL NULL NULL NULL +341 val_341 NULL NULL NULL NULL +342 val_342 NULL NULL NULL NULL +342 val_342 NULL NULL NULL NULL +344 val_344 NULL NULL NULL NULL +344 val_344 NULL NULL NULL NULL +345 val_345 NULL NULL NULL NULL +348 val_348 NULL NULL NULL NULL +348 val_348 NULL NULL NULL NULL +348 val_348 NULL NULL NULL NULL +348 val_348 NULL NULL NULL NULL +348 val_348 NULL NULL NULL NULL +35 val_35 NULL NULL NULL NULL +35 val_35 NULL NULL NULL NULL +35 val_35 NULL NULL NULL NULL +351 val_351 NULL NULL NULL NULL +353 val_353 NULL NULL NULL NULL +353 val_353 NULL NULL NULL NULL +356 val_356 NULL NULL NULL NULL +360 val_360 NULL NULL NULL NULL +362 val_362 NULL NULL NULL NULL +364 val_364 NULL NULL NULL NULL +365 val_365 NULL NULL NULL NULL +366 val_366 NULL NULL NULL NULL +367 val_367 NULL NULL NULL NULL +367 val_367 NULL NULL NULL NULL +368 val_368 NULL NULL NULL NULL +369 val_369 NULL NULL NULL NULL +369 val_369 NULL NULL NULL NULL +369 val_369 NULL NULL NULL NULL +37 val_37 NULL NULL NULL NULL +37 val_37 NULL NULL NULL NULL +373 val_373 NULL NULL NULL NULL +374 val_374 NULL NULL NULL NULL +375 val_375 NULL NULL NULL NULL +377 val_377 NULL NULL NULL NULL +378 val_378 NULL NULL NULL NULL +379 val_379 NULL NULL NULL NULL +382 val_382 NULL NULL NULL NULL +382 val_382 NULL NULL NULL NULL +384 val_384 NULL NULL NULL NULL +384 val_384 NULL NULL NULL NULL +384 val_384 NULL NULL NULL NULL +386 val_386 NULL NULL NULL NULL +389 val_389 NULL NULL NULL NULL +392 val_392 NULL NULL NULL NULL +393 val_393 NULL NULL NULL NULL +394 val_394 NULL NULL NULL NULL +395 val_395 NULL NULL NULL NULL +395 val_395 NULL NULL NULL NULL +396 val_396 NULL NULL NULL NULL +396 val_396 NULL NULL NULL NULL +396 val_396 NULL NULL NULL NULL +397 val_397 NULL NULL NULL NULL +397 val_397 NULL NULL NULL NULL +399 val_399 NULL NULL NULL NULL +399 val_399 NULL NULL NULL NULL +4 val_4 NULL NULL NULL NULL +400 val_400 NULL NULL NULL NULL +401 val_401 NULL NULL NULL NULL +401 val_401 NULL NULL NULL NULL +401 val_401 NULL NULL NULL NULL +401 val_401 NULL NULL NULL NULL +401 val_401 NULL NULL NULL NULL +402 val_402 NULL NULL NULL NULL +403 val_403 NULL NULL NULL NULL +403 val_403 NULL NULL NULL NULL +403 val_403 NULL NULL NULL NULL +404 val_404 NULL NULL NULL NULL +404 val_404 NULL NULL NULL NULL +406 val_406 NULL NULL NULL NULL +406 val_406 NULL NULL NULL NULL +406 val_406 NULL NULL NULL NULL +406 val_406 NULL NULL NULL NULL +407 val_407 NULL NULL NULL NULL +409 val_409 NULL NULL NULL NULL +409 val_409 NULL NULL NULL NULL +409 val_409 NULL NULL NULL NULL +41 val_41 NULL NULL NULL NULL +411 val_411 NULL NULL NULL NULL +413 val_413 NULL NULL NULL NULL +413 val_413 NULL NULL NULL NULL +414 val_414 NULL NULL NULL NULL +414 val_414 NULL NULL NULL NULL +417 val_417 NULL NULL NULL NULL +417 val_417 NULL NULL NULL NULL +417 val_417 NULL NULL NULL NULL +418 val_418 NULL NULL NULL NULL +419 val_419 NULL NULL NULL NULL +42 val_42 NULL NULL NULL NULL +42 val_42 NULL NULL NULL NULL +421 val_421 NULL NULL NULL NULL +424 val_424 NULL NULL NULL NULL +424 val_424 NULL NULL NULL NULL +427 val_427 NULL NULL NULL NULL +429 val_429 NULL NULL NULL NULL +429 val_429 NULL NULL NULL NULL +43 val_43 NULL NULL NULL NULL +430 val_430 NULL NULL NULL NULL +430 val_430 NULL NULL NULL NULL +430 val_430 NULL NULL NULL NULL +431 val_431 NULL NULL NULL NULL +431 val_431 NULL NULL NULL NULL +431 val_431 NULL NULL NULL NULL +432 val_432 NULL NULL NULL NULL +435 val_435 NULL NULL NULL NULL +436 val_436 NULL NULL NULL NULL +437 val_437 NULL NULL NULL NULL +438 val_438 NULL NULL NULL NULL +438 val_438 NULL NULL NULL NULL +438 val_438 NULL NULL NULL NULL +439 val_439 NULL NULL NULL NULL +439 val_439 NULL NULL NULL NULL +44 val_44 NULL NULL NULL NULL +443 val_443 NULL NULL NULL NULL +444 val_444 NULL NULL NULL NULL +446 val_446 NULL NULL NULL NULL +448 val_448 NULL NULL NULL NULL +449 val_449 NULL NULL NULL NULL +452 val_452 NULL NULL NULL NULL +453 val_453 NULL NULL NULL NULL +454 val_454 NULL NULL NULL NULL +454 val_454 NULL NULL NULL NULL +454 val_454 NULL NULL NULL NULL +455 val_455 NULL NULL NULL NULL +457 val_457 NULL NULL NULL NULL +458 val_458 NULL NULL NULL NULL +458 val_458 NULL NULL NULL NULL +459 val_459 NULL NULL NULL NULL +459 val_459 NULL NULL NULL NULL +460 val_460 NULL NULL NULL NULL +462 val_462 NULL NULL NULL NULL +462 val_462 NULL NULL NULL NULL +463 val_463 NULL NULL NULL NULL +463 val_463 NULL NULL NULL NULL +466 val_466 NULL NULL NULL NULL +466 val_466 NULL NULL NULL NULL +466 val_466 NULL NULL NULL NULL +467 val_467 NULL NULL NULL NULL +468 val_468 NULL NULL NULL NULL +468 val_468 NULL NULL NULL NULL +468 val_468 NULL NULL NULL NULL +468 val_468 NULL NULL NULL NULL +469 val_469 NULL NULL NULL NULL +469 val_469 NULL NULL NULL NULL +469 val_469 NULL NULL NULL NULL +469 val_469 NULL NULL NULL NULL +469 val_469 NULL NULL NULL NULL +47 val_47 NULL NULL NULL NULL +470 val_470 NULL NULL NULL NULL +472 val_472 NULL NULL NULL NULL +475 val_475 NULL NULL NULL NULL +477 val_477 NULL NULL NULL NULL +478 val_478 NULL NULL NULL NULL +478 val_478 NULL NULL NULL NULL +479 val_479 NULL NULL NULL NULL +480 val_480 NULL NULL NULL NULL +480 val_480 NULL NULL NULL NULL +480 val_480 NULL NULL NULL NULL +481 val_481 NULL NULL NULL NULL +482 val_482 NULL NULL NULL NULL +483 val_483 NULL NULL NULL NULL +484 val_484 NULL NULL NULL NULL +485 val_485 NULL NULL NULL NULL +487 val_487 NULL NULL NULL NULL +489 val_489 NULL NULL NULL NULL +489 val_489 NULL NULL NULL NULL +489 val_489 NULL NULL NULL NULL +489 val_489 NULL NULL NULL NULL +490 val_490 NULL NULL NULL NULL +491 val_491 NULL NULL NULL NULL +492 val_492 NULL NULL NULL NULL +492 val_492 NULL NULL NULL NULL +493 val_493 NULL NULL NULL NULL +494 val_494 NULL NULL NULL NULL +495 val_495 NULL NULL NULL NULL +496 val_496 NULL NULL NULL NULL +497 val_497 NULL NULL NULL NULL +498 val_498 NULL NULL NULL NULL +498 val_498 NULL NULL NULL NULL +498 val_498 NULL NULL NULL NULL +5 val_5 NULL NULL NULL NULL +5 val_5 NULL NULL NULL NULL +5 val_5 NULL NULL NULL NULL +51 val_51 NULL NULL NULL NULL +51 val_51 NULL NULL NULL NULL +53 val_53 NULL NULL NULL NULL +54 val_54 NULL NULL NULL NULL +57 val_57 NULL NULL NULL NULL +58 val_58 NULL NULL NULL NULL +58 val_58 NULL NULL NULL NULL +64 val_64 NULL NULL NULL NULL +65 val_65 NULL NULL NULL NULL +66 val_66 NULL NULL NULL NULL +67 val_67 NULL NULL NULL NULL +67 val_67 NULL NULL NULL NULL +69 val_69 NULL NULL NULL NULL +70 val_70 NULL NULL NULL NULL +70 val_70 NULL NULL NULL NULL +70 val_70 NULL NULL NULL NULL +72 val_72 NULL NULL NULL NULL +72 val_72 NULL NULL NULL NULL +74 val_74 NULL NULL NULL NULL +76 val_76 NULL NULL NULL NULL +76 val_76 NULL NULL NULL NULL +77 val_77 NULL NULL NULL NULL +78 val_78 NULL NULL NULL NULL +8 val_8 NULL NULL NULL NULL +80 val_80 NULL NULL NULL NULL +82 val_82 NULL NULL NULL NULL +83 val_83 NULL NULL NULL NULL +83 val_83 NULL NULL NULL NULL +84 val_84 NULL NULL NULL NULL +84 val_84 NULL NULL NULL NULL +85 val_85 NULL NULL NULL NULL +86 val_86 NULL NULL NULL NULL +87 val_87 NULL NULL NULL NULL +9 val_9 NULL NULL NULL NULL +90 val_90 NULL NULL NULL NULL +90 val_90 NULL NULL NULL NULL +90 val_90 NULL NULL NULL NULL +92 val_92 NULL NULL NULL NULL +95 val_95 NULL NULL NULL NULL +95 val_95 NULL NULL NULL NULL +96 val_96 NULL NULL NULL NULL +97 val_97 NULL NULL NULL NULL +97 val_97 NULL NULL NULL NULL +98 val_98 NULL NULL NULL NULL +98 val_98 NULL NULL NULL NULL +PREHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + filter predicates: + 0 + 1 {(KEY.reducesinkkey0 > 10)} + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 10 val_10 NULL NULL +NULL NULL 100 val_100 NULL NULL +NULL NULL 100 val_100 NULL NULL +NULL NULL 103 val_103 NULL NULL +NULL NULL 103 val_103 NULL NULL +NULL NULL 104 val_104 NULL NULL +NULL NULL 104 val_104 NULL NULL +NULL NULL 105 val_105 NULL NULL +NULL NULL 11 val_11 NULL NULL +NULL NULL 111 val_111 NULL NULL +NULL NULL 113 val_113 NULL NULL +NULL NULL 113 val_113 NULL NULL +NULL NULL 114 val_114 NULL NULL +NULL NULL 116 val_116 NULL NULL +NULL NULL 118 val_118 NULL NULL +NULL NULL 118 val_118 NULL NULL +NULL NULL 119 val_119 NULL NULL +NULL NULL 119 val_119 NULL NULL +NULL NULL 119 val_119 NULL NULL +NULL NULL 12 val_12 NULL NULL +NULL NULL 12 val_12 NULL NULL +NULL NULL 120 val_120 NULL NULL +NULL NULL 120 val_120 NULL NULL +NULL NULL 125 val_125 NULL NULL +NULL NULL 125 val_125 NULL NULL +NULL NULL 126 val_126 NULL NULL +NULL NULL 128 val_128 NULL NULL +NULL NULL 128 val_128 NULL NULL +NULL NULL 128 val_128 NULL NULL +NULL NULL 129 val_129 NULL NULL +NULL NULL 129 val_129 NULL NULL +NULL NULL 131 val_131 NULL NULL +NULL NULL 133 val_133 NULL NULL +NULL NULL 134 val_134 NULL NULL +NULL NULL 134 val_134 NULL NULL +NULL NULL 136 val_136 NULL NULL +NULL NULL 137 val_137 NULL NULL +NULL NULL 137 val_137 NULL NULL +NULL NULL 138 val_138 NULL NULL +NULL NULL 138 val_138 NULL NULL +NULL NULL 138 val_138 NULL NULL +NULL NULL 138 val_138 NULL NULL +NULL NULL 143 val_143 NULL NULL +NULL NULL 145 val_145 NULL NULL +NULL NULL 146 val_146 NULL NULL +NULL NULL 146 val_146 NULL NULL +NULL NULL 149 val_149 NULL NULL +NULL NULL 149 val_149 NULL NULL +NULL NULL 15 val_15 NULL NULL +NULL NULL 15 val_15 NULL NULL +NULL NULL 150 val_150 NULL NULL +NULL NULL 152 val_152 NULL NULL +NULL NULL 152 val_152 NULL NULL +NULL NULL 153 val_153 NULL NULL +NULL NULL 155 val_155 NULL NULL +NULL NULL 156 val_156 NULL NULL +NULL NULL 157 val_157 NULL NULL +NULL NULL 158 val_158 NULL NULL +NULL NULL 160 val_160 NULL NULL +NULL NULL 162 val_162 NULL NULL +NULL NULL 163 val_163 NULL NULL +NULL NULL 164 val_164 NULL NULL +NULL NULL 164 val_164 NULL NULL +NULL NULL 165 val_165 NULL NULL +NULL NULL 165 val_165 NULL NULL +NULL NULL 166 val_166 NULL NULL +NULL NULL 167 val_167 NULL NULL +NULL NULL 167 val_167 NULL NULL +NULL NULL 167 val_167 NULL NULL +NULL NULL 168 val_168 NULL NULL +NULL NULL 169 val_169 NULL NULL +NULL NULL 169 val_169 NULL NULL +NULL NULL 169 val_169 NULL NULL +NULL NULL 169 val_169 NULL NULL +NULL NULL 17 val_17 NULL NULL +NULL NULL 170 val_170 NULL NULL +NULL NULL 172 val_172 NULL NULL +NULL NULL 172 val_172 NULL NULL +NULL NULL 174 val_174 NULL NULL +NULL NULL 174 val_174 NULL NULL +NULL NULL 175 val_175 NULL NULL +NULL NULL 175 val_175 NULL NULL +NULL NULL 176 val_176 NULL NULL +NULL NULL 176 val_176 NULL NULL +NULL NULL 177 val_177 NULL NULL +NULL NULL 178 val_178 NULL NULL +NULL NULL 179 val_179 NULL NULL +NULL NULL 179 val_179 NULL NULL +NULL NULL 18 val_18 NULL NULL +NULL NULL 18 val_18 NULL NULL +NULL NULL 180 val_180 NULL NULL +NULL NULL 181 val_181 NULL NULL +NULL NULL 183 val_183 NULL NULL +NULL NULL 186 val_186 NULL NULL +NULL NULL 187 val_187 NULL NULL +NULL NULL 187 val_187 NULL NULL +NULL NULL 187 val_187 NULL NULL +NULL NULL 189 val_189 NULL NULL +NULL NULL 19 val_19 NULL NULL +NULL NULL 190 val_190 NULL NULL +NULL NULL 191 val_191 NULL NULL +NULL NULL 191 val_191 NULL NULL +NULL NULL 192 val_192 NULL NULL +NULL NULL 193 val_193 NULL NULL +NULL NULL 193 val_193 NULL NULL +NULL NULL 193 val_193 NULL NULL +NULL NULL 194 val_194 NULL NULL +NULL NULL 195 val_195 NULL NULL +NULL NULL 195 val_195 NULL NULL +NULL NULL 196 val_196 NULL NULL +NULL NULL 197 val_197 NULL NULL +NULL NULL 197 val_197 NULL NULL +NULL NULL 199 val_199 NULL NULL +NULL NULL 199 val_199 NULL NULL +NULL NULL 199 val_199 NULL NULL +NULL NULL 2 val_2 2 val_2 +NULL NULL 20 val_20 NULL NULL +NULL NULL 200 val_200 NULL NULL +NULL NULL 200 val_200 NULL NULL +NULL NULL 201 val_201 NULL NULL +NULL NULL 202 val_202 NULL NULL +NULL NULL 203 val_203 NULL NULL +NULL NULL 203 val_203 NULL NULL +NULL NULL 205 val_205 NULL NULL +NULL NULL 205 val_205 NULL NULL +NULL NULL 207 val_207 NULL NULL +NULL NULL 207 val_207 NULL NULL +NULL NULL 208 val_208 NULL NULL +NULL NULL 208 val_208 NULL NULL +NULL NULL 208 val_208 NULL NULL +NULL NULL 209 val_209 NULL NULL +NULL NULL 209 val_209 NULL NULL +NULL NULL 213 val_213 NULL NULL +NULL NULL 213 val_213 NULL NULL +NULL NULL 214 val_214 NULL NULL +NULL NULL 216 val_216 NULL NULL +NULL NULL 216 val_216 NULL NULL +NULL NULL 217 val_217 NULL NULL +NULL NULL 217 val_217 NULL NULL +NULL NULL 218 val_218 NULL NULL +NULL NULL 219 val_219 NULL NULL +NULL NULL 219 val_219 NULL NULL +NULL NULL 221 val_221 NULL NULL +NULL NULL 221 val_221 NULL NULL +NULL NULL 222 val_222 NULL NULL +NULL NULL 223 val_223 NULL NULL +NULL NULL 223 val_223 NULL NULL +NULL NULL 224 val_224 NULL NULL +NULL NULL 224 val_224 NULL NULL +NULL NULL 226 val_226 NULL NULL +NULL NULL 228 val_228 NULL NULL +NULL NULL 229 val_229 NULL NULL +NULL NULL 229 val_229 NULL NULL +NULL NULL 230 val_230 NULL NULL +NULL NULL 230 val_230 NULL NULL +NULL NULL 230 val_230 NULL NULL +NULL NULL 230 val_230 NULL NULL +NULL NULL 230 val_230 NULL NULL +NULL NULL 233 val_233 NULL NULL +NULL NULL 233 val_233 NULL NULL +NULL NULL 235 val_235 NULL NULL +NULL NULL 237 val_237 NULL NULL +NULL NULL 237 val_237 NULL NULL +NULL NULL 238 val_238 NULL NULL +NULL NULL 238 val_238 NULL NULL +NULL NULL 239 val_239 NULL NULL +NULL NULL 239 val_239 NULL NULL +NULL NULL 24 val_24 NULL NULL +NULL NULL 24 val_24 NULL NULL +NULL NULL 241 val_241 NULL NULL +NULL NULL 242 val_242 NULL NULL +NULL NULL 242 val_242 NULL NULL +NULL NULL 244 val_244 NULL NULL +NULL NULL 247 val_247 NULL NULL +NULL NULL 248 val_248 NULL NULL +NULL NULL 249 val_249 NULL NULL +NULL NULL 252 val_252 NULL NULL +NULL NULL 255 val_255 NULL NULL +NULL NULL 255 val_255 NULL NULL +NULL NULL 256 val_256 NULL NULL +NULL NULL 256 val_256 NULL NULL +NULL NULL 257 val_257 NULL NULL +NULL NULL 258 val_258 NULL NULL +NULL NULL 26 val_26 NULL NULL +NULL NULL 26 val_26 NULL NULL +NULL NULL 260 val_260 NULL NULL +NULL NULL 262 val_262 NULL NULL +NULL NULL 263 val_263 NULL NULL +NULL NULL 265 val_265 NULL NULL +NULL NULL 265 val_265 NULL NULL +NULL NULL 266 val_266 NULL NULL +NULL NULL 27 val_27 NULL NULL +NULL NULL 272 val_272 NULL NULL +NULL NULL 272 val_272 NULL NULL +NULL NULL 273 val_273 NULL NULL +NULL NULL 273 val_273 NULL NULL +NULL NULL 273 val_273 NULL NULL +NULL NULL 274 val_274 NULL NULL +NULL NULL 275 val_275 NULL NULL +NULL NULL 277 val_277 NULL NULL +NULL NULL 277 val_277 NULL NULL +NULL NULL 277 val_277 NULL NULL +NULL NULL 277 val_277 NULL NULL +NULL NULL 278 val_278 NULL NULL +NULL NULL 278 val_278 NULL NULL +NULL NULL 28 val_28 NULL NULL +NULL NULL 280 val_280 NULL NULL +NULL NULL 280 val_280 NULL NULL +NULL NULL 281 val_281 NULL NULL +NULL NULL 281 val_281 NULL NULL +NULL NULL 282 val_282 NULL NULL +NULL NULL 282 val_282 NULL NULL +NULL NULL 283 val_283 NULL NULL +NULL NULL 284 val_284 NULL NULL +NULL NULL 285 val_285 NULL NULL +NULL NULL 286 val_286 NULL NULL +NULL NULL 287 val_287 NULL NULL +NULL NULL 288 val_288 NULL NULL +NULL NULL 288 val_288 NULL NULL +NULL NULL 289 val_289 NULL NULL +NULL NULL 291 val_291 NULL NULL +NULL NULL 292 val_292 NULL NULL +NULL NULL 296 val_296 NULL NULL +NULL NULL 298 val_298 NULL NULL +NULL NULL 298 val_298 NULL NULL +NULL NULL 298 val_298 NULL NULL +NULL NULL 30 val_30 NULL NULL +NULL NULL 302 val_302 NULL NULL +NULL NULL 305 val_305 NULL NULL +NULL NULL 306 val_306 NULL NULL +NULL NULL 307 val_307 NULL NULL +NULL NULL 307 val_307 NULL NULL +NULL NULL 308 val_308 NULL NULL +NULL NULL 309 val_309 NULL NULL +NULL NULL 309 val_309 NULL NULL +NULL NULL 310 val_310 NULL NULL +NULL NULL 311 val_311 NULL NULL +NULL NULL 311 val_311 NULL NULL +NULL NULL 311 val_311 NULL NULL +NULL NULL 315 val_315 NULL NULL +NULL NULL 316 val_316 NULL NULL +NULL NULL 316 val_316 NULL NULL +NULL NULL 316 val_316 NULL NULL +NULL NULL 317 val_317 NULL NULL +NULL NULL 317 val_317 NULL NULL +NULL NULL 318 val_318 NULL NULL +NULL NULL 318 val_318 NULL NULL +NULL NULL 318 val_318 NULL NULL +NULL NULL 321 val_321 NULL NULL +NULL NULL 321 val_321 NULL NULL +NULL NULL 322 val_322 NULL NULL +NULL NULL 322 val_322 NULL NULL +NULL NULL 323 val_323 NULL NULL +NULL NULL 325 val_325 NULL NULL +NULL NULL 325 val_325 NULL NULL +NULL NULL 327 val_327 NULL NULL +NULL NULL 327 val_327 NULL NULL +NULL NULL 327 val_327 NULL NULL +NULL NULL 33 val_33 NULL NULL +NULL NULL 331 val_331 NULL NULL +NULL NULL 331 val_331 NULL NULL +NULL NULL 332 val_332 NULL NULL +NULL NULL 333 val_333 NULL NULL +NULL NULL 333 val_333 NULL NULL +NULL NULL 335 val_335 NULL NULL +NULL NULL 336 val_336 NULL NULL +NULL NULL 338 val_338 NULL NULL +NULL NULL 339 val_339 NULL NULL +NULL NULL 34 val_34 NULL NULL +NULL NULL 341 val_341 NULL NULL +NULL NULL 342 val_342 NULL NULL +NULL NULL 342 val_342 NULL NULL +NULL NULL 344 val_344 NULL NULL +NULL NULL 344 val_344 NULL NULL +NULL NULL 345 val_345 NULL NULL +NULL NULL 348 val_348 NULL NULL +NULL NULL 348 val_348 NULL NULL +NULL NULL 348 val_348 NULL NULL +NULL NULL 348 val_348 NULL NULL +NULL NULL 348 val_348 NULL NULL +NULL NULL 35 val_35 NULL NULL +NULL NULL 35 val_35 NULL NULL +NULL NULL 35 val_35 NULL NULL +NULL NULL 351 val_351 NULL NULL +NULL NULL 353 val_353 NULL NULL +NULL NULL 353 val_353 NULL NULL +NULL NULL 356 val_356 NULL NULL +NULL NULL 360 val_360 NULL NULL +NULL NULL 362 val_362 NULL NULL +NULL NULL 364 val_364 NULL NULL +NULL NULL 365 val_365 NULL NULL +NULL NULL 366 val_366 NULL NULL +NULL NULL 367 val_367 NULL NULL +NULL NULL 367 val_367 NULL NULL +NULL NULL 368 val_368 NULL NULL +NULL NULL 369 val_369 NULL NULL +NULL NULL 369 val_369 NULL NULL +NULL NULL 369 val_369 NULL NULL +NULL NULL 37 val_37 NULL NULL +NULL NULL 37 val_37 NULL NULL +NULL NULL 373 val_373 NULL NULL +NULL NULL 374 val_374 NULL NULL +NULL NULL 375 val_375 NULL NULL +NULL NULL 377 val_377 NULL NULL +NULL NULL 378 val_378 NULL NULL +NULL NULL 379 val_379 NULL NULL +NULL NULL 382 val_382 NULL NULL +NULL NULL 382 val_382 NULL NULL +NULL NULL 384 val_384 NULL NULL +NULL NULL 384 val_384 NULL NULL +NULL NULL 384 val_384 NULL NULL +NULL NULL 386 val_386 NULL NULL +NULL NULL 389 val_389 NULL NULL +NULL NULL 392 val_392 NULL NULL +NULL NULL 393 val_393 NULL NULL +NULL NULL 394 val_394 NULL NULL +NULL NULL 395 val_395 NULL NULL +NULL NULL 395 val_395 NULL NULL +NULL NULL 396 val_396 NULL NULL +NULL NULL 396 val_396 NULL NULL +NULL NULL 396 val_396 NULL NULL +NULL NULL 397 val_397 NULL NULL +NULL NULL 397 val_397 NULL NULL +NULL NULL 399 val_399 NULL NULL +NULL NULL 399 val_399 NULL NULL +NULL NULL 4 val_4 4 val_4 +NULL NULL 400 val_400 NULL NULL +NULL NULL 401 val_401 NULL NULL +NULL NULL 401 val_401 NULL NULL +NULL NULL 401 val_401 NULL NULL +NULL NULL 401 val_401 NULL NULL +NULL NULL 401 val_401 NULL NULL +NULL NULL 402 val_402 NULL NULL +NULL NULL 403 val_403 NULL NULL +NULL NULL 403 val_403 NULL NULL +NULL NULL 403 val_403 NULL NULL +NULL NULL 404 val_404 NULL NULL +NULL NULL 404 val_404 NULL NULL +NULL NULL 406 val_406 NULL NULL +NULL NULL 406 val_406 NULL NULL +NULL NULL 406 val_406 NULL NULL +NULL NULL 406 val_406 NULL NULL +NULL NULL 407 val_407 NULL NULL +NULL NULL 409 val_409 NULL NULL +NULL NULL 409 val_409 NULL NULL +NULL NULL 409 val_409 NULL NULL +NULL NULL 41 val_41 NULL NULL +NULL NULL 411 val_411 NULL NULL +NULL NULL 413 val_413 NULL NULL +NULL NULL 413 val_413 NULL NULL +NULL NULL 414 val_414 NULL NULL +NULL NULL 414 val_414 NULL NULL +NULL NULL 417 val_417 NULL NULL +NULL NULL 417 val_417 NULL NULL +NULL NULL 417 val_417 NULL NULL +NULL NULL 418 val_418 NULL NULL +NULL NULL 419 val_419 NULL NULL +NULL NULL 42 val_42 NULL NULL +NULL NULL 42 val_42 NULL NULL +NULL NULL 421 val_421 NULL NULL +NULL NULL 424 val_424 NULL NULL +NULL NULL 424 val_424 NULL NULL +NULL NULL 427 val_427 NULL NULL +NULL NULL 429 val_429 NULL NULL +NULL NULL 429 val_429 NULL NULL +NULL NULL 43 val_43 NULL NULL +NULL NULL 430 val_430 NULL NULL +NULL NULL 430 val_430 NULL NULL +NULL NULL 430 val_430 NULL NULL +NULL NULL 431 val_431 NULL NULL +NULL NULL 431 val_431 NULL NULL +NULL NULL 431 val_431 NULL NULL +NULL NULL 432 val_432 NULL NULL +NULL NULL 435 val_435 NULL NULL +NULL NULL 436 val_436 NULL NULL +NULL NULL 437 val_437 NULL NULL +NULL NULL 438 val_438 NULL NULL +NULL NULL 438 val_438 NULL NULL +NULL NULL 438 val_438 NULL NULL +NULL NULL 439 val_439 NULL NULL +NULL NULL 439 val_439 NULL NULL +NULL NULL 44 val_44 NULL NULL +NULL NULL 443 val_443 NULL NULL +NULL NULL 444 val_444 NULL NULL +NULL NULL 446 val_446 NULL NULL +NULL NULL 448 val_448 NULL NULL +NULL NULL 449 val_449 NULL NULL +NULL NULL 452 val_452 NULL NULL +NULL NULL 453 val_453 NULL NULL +NULL NULL 454 val_454 NULL NULL +NULL NULL 454 val_454 NULL NULL +NULL NULL 454 val_454 NULL NULL +NULL NULL 455 val_455 NULL NULL +NULL NULL 457 val_457 NULL NULL +NULL NULL 458 val_458 NULL NULL +NULL NULL 458 val_458 NULL NULL +NULL NULL 459 val_459 NULL NULL +NULL NULL 459 val_459 NULL NULL +NULL NULL 460 val_460 NULL NULL +NULL NULL 462 val_462 NULL NULL +NULL NULL 462 val_462 NULL NULL +NULL NULL 463 val_463 NULL NULL +NULL NULL 463 val_463 NULL NULL +NULL NULL 466 val_466 NULL NULL +NULL NULL 466 val_466 NULL NULL +NULL NULL 466 val_466 NULL NULL +NULL NULL 467 val_467 NULL NULL +NULL NULL 468 val_468 NULL NULL +NULL NULL 468 val_468 NULL NULL +NULL NULL 468 val_468 NULL NULL +NULL NULL 468 val_468 NULL NULL +NULL NULL 469 val_469 NULL NULL +NULL NULL 469 val_469 NULL NULL +NULL NULL 469 val_469 NULL NULL +NULL NULL 469 val_469 NULL NULL +NULL NULL 469 val_469 NULL NULL +NULL NULL 47 val_47 NULL NULL +NULL NULL 470 val_470 NULL NULL +NULL NULL 472 val_472 NULL NULL +NULL NULL 475 val_475 NULL NULL +NULL NULL 477 val_477 NULL NULL +NULL NULL 478 val_478 NULL NULL +NULL NULL 478 val_478 NULL NULL +NULL NULL 479 val_479 NULL NULL +NULL NULL 480 val_480 NULL NULL +NULL NULL 480 val_480 NULL NULL +NULL NULL 480 val_480 NULL NULL +NULL NULL 481 val_481 NULL NULL +NULL NULL 482 val_482 NULL NULL +NULL NULL 483 val_483 NULL NULL +NULL NULL 484 val_484 NULL NULL +NULL NULL 485 val_485 NULL NULL +NULL NULL 487 val_487 NULL NULL +NULL NULL 489 val_489 NULL NULL +NULL NULL 489 val_489 NULL NULL +NULL NULL 489 val_489 NULL NULL +NULL NULL 489 val_489 NULL NULL +NULL NULL 490 val_490 NULL NULL +NULL NULL 491 val_491 NULL NULL +NULL NULL 492 val_492 NULL NULL +NULL NULL 492 val_492 NULL NULL +NULL NULL 493 val_493 NULL NULL +NULL NULL 494 val_494 NULL NULL +NULL NULL 495 val_495 NULL NULL +NULL NULL 496 val_496 NULL NULL +NULL NULL 497 val_497 NULL NULL +NULL NULL 498 val_498 NULL NULL +NULL NULL 498 val_498 NULL NULL +NULL NULL 498 val_498 NULL NULL +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 51 val_51 NULL NULL +NULL NULL 51 val_51 NULL NULL +NULL NULL 53 val_53 NULL NULL +NULL NULL 54 val_54 NULL NULL +NULL NULL 57 val_57 NULL NULL +NULL NULL 58 val_58 NULL NULL +NULL NULL 58 val_58 NULL NULL +NULL NULL 64 val_64 NULL NULL +NULL NULL 65 val_65 NULL NULL +NULL NULL 66 val_66 NULL NULL +NULL NULL 67 val_67 NULL NULL +NULL NULL 67 val_67 NULL NULL +NULL NULL 69 val_69 NULL NULL +NULL NULL 70 val_70 NULL NULL +NULL NULL 70 val_70 NULL NULL +NULL NULL 70 val_70 NULL NULL +NULL NULL 72 val_72 NULL NULL +NULL NULL 72 val_72 NULL NULL +NULL NULL 74 val_74 NULL NULL +NULL NULL 76 val_76 NULL NULL +NULL NULL 76 val_76 NULL NULL +NULL NULL 77 val_77 NULL NULL +NULL NULL 78 val_78 NULL NULL +NULL NULL 8 val_8 8 val_8 +NULL NULL 80 val_80 NULL NULL +NULL NULL 82 val_82 NULL NULL +NULL NULL 83 val_83 NULL NULL +NULL NULL 83 val_83 NULL NULL +NULL NULL 84 val_84 NULL NULL +NULL NULL 84 val_84 NULL NULL +NULL NULL 85 val_85 NULL NULL +NULL NULL 86 val_86 NULL NULL +NULL NULL 87 val_87 NULL NULL +NULL NULL 9 val_9 9 val_9 +NULL NULL 90 val_90 NULL NULL +NULL NULL 90 val_90 NULL NULL +NULL NULL 90 val_90 NULL NULL +NULL NULL 92 val_92 NULL NULL +NULL NULL 95 val_95 NULL NULL +NULL NULL 95 val_95 NULL NULL +NULL NULL 96 val_96 NULL NULL +NULL NULL 97 val_97 NULL NULL +NULL NULL 97 val_97 NULL NULL +NULL NULL 98 val_98 NULL NULL +NULL NULL 98 val_98 NULL NULL +PREHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Right Outer Join1 to 2 + filter predicates: + 0 + 1 {(KEY.reducesinkkey0 > 10)} + 2 {(KEY.reducesinkkey0 < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 2 val_2 2 val_2 +NULL NULL 4 val_4 4 val_4 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 8 val_8 8 val_8 +NULL NULL 9 val_9 9 val_9 +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 100 val_100 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 103 val_103 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 104 val_104 +NULL NULL NULL NULL 105 val_105 +NULL NULL NULL NULL 11 val_11 +NULL NULL NULL NULL 111 val_111 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 113 val_113 +NULL NULL NULL NULL 114 val_114 +NULL NULL NULL NULL 116 val_116 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 118 val_118 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 119 val_119 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 12 val_12 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 120 val_120 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 125 val_125 +NULL NULL NULL NULL 126 val_126 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 128 val_128 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 129 val_129 +NULL NULL NULL NULL 131 val_131 +NULL NULL NULL NULL 133 val_133 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 134 val_134 +NULL NULL NULL NULL 136 val_136 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 137 val_137 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 138 val_138 +NULL NULL NULL NULL 143 val_143 +NULL NULL NULL NULL 145 val_145 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 146 val_146 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 149 val_149 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 15 val_15 +NULL NULL NULL NULL 150 val_150 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 152 val_152 +NULL NULL NULL NULL 153 val_153 +NULL NULL NULL NULL 155 val_155 +NULL NULL NULL NULL 156 val_156 +NULL NULL NULL NULL 157 val_157 +NULL NULL NULL NULL 158 val_158 +NULL NULL NULL NULL 160 val_160 +NULL NULL NULL NULL 162 val_162 +NULL NULL NULL NULL 163 val_163 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 164 val_164 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 165 val_165 +NULL NULL NULL NULL 166 val_166 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 167 val_167 +NULL NULL NULL NULL 168 val_168 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 169 val_169 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 170 val_170 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 172 val_172 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 174 val_174 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 175 val_175 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 176 val_176 +NULL NULL NULL NULL 177 val_177 +NULL NULL NULL NULL 178 val_178 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 179 val_179 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 18 val_18 +NULL NULL NULL NULL 180 val_180 +NULL NULL NULL NULL 181 val_181 +NULL NULL NULL NULL 183 val_183 +NULL NULL NULL NULL 186 val_186 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 187 val_187 +NULL NULL NULL NULL 189 val_189 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 190 val_190 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 191 val_191 +NULL NULL NULL NULL 192 val_192 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 193 val_193 +NULL NULL NULL NULL 194 val_194 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 195 val_195 +NULL NULL NULL NULL 196 val_196 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 197 val_197 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 199 val_199 +NULL NULL NULL NULL 20 val_20 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 200 val_200 +NULL NULL NULL NULL 201 val_201 +NULL NULL NULL NULL 202 val_202 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 203 val_203 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 205 val_205 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 207 val_207 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 208 val_208 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 209 val_209 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 213 val_213 +NULL NULL NULL NULL 214 val_214 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 216 val_216 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 217 val_217 +NULL NULL NULL NULL 218 val_218 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 219 val_219 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 221 val_221 +NULL NULL NULL NULL 222 val_222 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 223 val_223 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 224 val_224 +NULL NULL NULL NULL 226 val_226 +NULL NULL NULL NULL 228 val_228 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 229 val_229 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 230 val_230 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 233 val_233 +NULL NULL NULL NULL 235 val_235 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 237 val_237 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 238 val_238 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 239 val_239 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 24 val_24 +NULL NULL NULL NULL 241 val_241 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 242 val_242 +NULL NULL NULL NULL 244 val_244 +NULL NULL NULL NULL 247 val_247 +NULL NULL NULL NULL 248 val_248 +NULL NULL NULL NULL 249 val_249 +NULL NULL NULL NULL 252 val_252 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 255 val_255 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 256 val_256 +NULL NULL NULL NULL 257 val_257 +NULL NULL NULL NULL 258 val_258 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 26 val_26 +NULL NULL NULL NULL 260 val_260 +NULL NULL NULL NULL 262 val_262 +NULL NULL NULL NULL 263 val_263 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 265 val_265 +NULL NULL NULL NULL 266 val_266 +NULL NULL NULL NULL 27 val_27 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 272 val_272 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 273 val_273 +NULL NULL NULL NULL 274 val_274 +NULL NULL NULL NULL 275 val_275 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 277 val_277 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 278 val_278 +NULL NULL NULL NULL 28 val_28 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 280 val_280 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 281 val_281 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 282 val_282 +NULL NULL NULL NULL 283 val_283 +NULL NULL NULL NULL 284 val_284 +NULL NULL NULL NULL 285 val_285 +NULL NULL NULL NULL 286 val_286 +NULL NULL NULL NULL 287 val_287 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 288 val_288 +NULL NULL NULL NULL 289 val_289 +NULL NULL NULL NULL 291 val_291 +NULL NULL NULL NULL 292 val_292 +NULL NULL NULL NULL 296 val_296 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 298 val_298 +NULL NULL NULL NULL 30 val_30 +NULL NULL NULL NULL 302 val_302 +NULL NULL NULL NULL 305 val_305 +NULL NULL NULL NULL 306 val_306 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 307 val_307 +NULL NULL NULL NULL 308 val_308 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 309 val_309 +NULL NULL NULL NULL 310 val_310 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 311 val_311 +NULL NULL NULL NULL 315 val_315 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 316 val_316 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 317 val_317 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 318 val_318 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 321 val_321 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 322 val_322 +NULL NULL NULL NULL 323 val_323 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 325 val_325 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 327 val_327 +NULL NULL NULL NULL 33 val_33 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 331 val_331 +NULL NULL NULL NULL 332 val_332 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 333 val_333 +NULL NULL NULL NULL 335 val_335 +NULL NULL NULL NULL 336 val_336 +NULL NULL NULL NULL 338 val_338 +NULL NULL NULL NULL 339 val_339 +NULL NULL NULL NULL 34 val_34 +NULL NULL NULL NULL 341 val_341 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 342 val_342 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 344 val_344 +NULL NULL NULL NULL 345 val_345 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 348 val_348 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 35 val_35 +NULL NULL NULL NULL 351 val_351 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 353 val_353 +NULL NULL NULL NULL 356 val_356 +NULL NULL NULL NULL 360 val_360 +NULL NULL NULL NULL 362 val_362 +NULL NULL NULL NULL 364 val_364 +NULL NULL NULL NULL 365 val_365 +NULL NULL NULL NULL 366 val_366 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 367 val_367 +NULL NULL NULL NULL 368 val_368 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 369 val_369 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 37 val_37 +NULL NULL NULL NULL 373 val_373 +NULL NULL NULL NULL 374 val_374 +NULL NULL NULL NULL 375 val_375 +NULL NULL NULL NULL 377 val_377 +NULL NULL NULL NULL 378 val_378 +NULL NULL NULL NULL 379 val_379 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 382 val_382 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 384 val_384 +NULL NULL NULL NULL 386 val_386 +NULL NULL NULL NULL 389 val_389 +NULL NULL NULL NULL 392 val_392 +NULL NULL NULL NULL 393 val_393 +NULL NULL NULL NULL 394 val_394 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 395 val_395 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 396 val_396 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 397 val_397 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 399 val_399 +NULL NULL NULL NULL 400 val_400 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 401 val_401 +NULL NULL NULL NULL 402 val_402 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 403 val_403 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 404 val_404 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 406 val_406 +NULL NULL NULL NULL 407 val_407 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 409 val_409 +NULL NULL NULL NULL 41 val_41 +NULL NULL NULL NULL 411 val_411 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 413 val_413 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 414 val_414 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 417 val_417 +NULL NULL NULL NULL 418 val_418 +NULL NULL NULL NULL 419 val_419 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 42 val_42 +NULL NULL NULL NULL 421 val_421 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 424 val_424 +NULL NULL NULL NULL 427 val_427 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 429 val_429 +NULL NULL NULL NULL 43 val_43 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 430 val_430 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 431 val_431 +NULL NULL NULL NULL 432 val_432 +NULL NULL NULL NULL 435 val_435 +NULL NULL NULL NULL 436 val_436 +NULL NULL NULL NULL 437 val_437 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 438 val_438 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 439 val_439 +NULL NULL NULL NULL 44 val_44 +NULL NULL NULL NULL 443 val_443 +NULL NULL NULL NULL 444 val_444 +NULL NULL NULL NULL 446 val_446 +NULL NULL NULL NULL 448 val_448 +NULL NULL NULL NULL 449 val_449 +NULL NULL NULL NULL 452 val_452 +NULL NULL NULL NULL 453 val_453 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 454 val_454 +NULL NULL NULL NULL 455 val_455 +NULL NULL NULL NULL 457 val_457 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 458 val_458 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 459 val_459 +NULL NULL NULL NULL 460 val_460 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 462 val_462 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 463 val_463 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 466 val_466 +NULL NULL NULL NULL 467 val_467 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 468 val_468 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 469 val_469 +NULL NULL NULL NULL 47 val_47 +NULL NULL NULL NULL 470 val_470 +NULL NULL NULL NULL 472 val_472 +NULL NULL NULL NULL 475 val_475 +NULL NULL NULL NULL 477 val_477 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 478 val_478 +NULL NULL NULL NULL 479 val_479 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 480 val_480 +NULL NULL NULL NULL 481 val_481 +NULL NULL NULL NULL 482 val_482 +NULL NULL NULL NULL 483 val_483 +NULL NULL NULL NULL 484 val_484 +NULL NULL NULL NULL 485 val_485 +NULL NULL NULL NULL 487 val_487 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 489 val_489 +NULL NULL NULL NULL 490 val_490 +NULL NULL NULL NULL 491 val_491 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 492 val_492 +NULL NULL NULL NULL 493 val_493 +NULL NULL NULL NULL 494 val_494 +NULL NULL NULL NULL 495 val_495 +NULL NULL NULL NULL 496 val_496 +NULL NULL NULL NULL 497 val_497 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 498 val_498 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 51 val_51 +NULL NULL NULL NULL 53 val_53 +NULL NULL NULL NULL 54 val_54 +NULL NULL NULL NULL 57 val_57 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 58 val_58 +NULL NULL NULL NULL 64 val_64 +NULL NULL NULL NULL 65 val_65 +NULL NULL NULL NULL 66 val_66 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 67 val_67 +NULL NULL NULL NULL 69 val_69 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 70 val_70 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 72 val_72 +NULL NULL NULL NULL 74 val_74 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 76 val_76 +NULL NULL NULL NULL 77 val_77 +NULL NULL NULL NULL 78 val_78 +NULL NULL NULL NULL 80 val_80 +NULL NULL NULL NULL 82 val_82 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 83 val_83 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 84 val_84 +NULL NULL NULL NULL 85 val_85 +NULL NULL NULL NULL 86 val_86 +NULL NULL NULL NULL 87 val_87 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 90 val_90 +NULL NULL NULL NULL 92 val_92 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 95 val_95 +NULL NULL NULL NULL 96 val_96 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 97 val_97 +NULL NULL NULL NULL 98 val_98 +NULL NULL NULL NULL 98 val_98 +PREHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Right Outer Join1 to 2 + filter predicates: + 0 + 1 + 2 {(KEY.reducesinkkey0 < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + filter predicates: + 0 {(key < 10)} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + input vertices: + 1 Map 3 + 2 Map 4 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 2 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + filter predicates: + 0 + 1 {(key > 10)} + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + input vertices: + 0 Map 1 + 2 Map 4 + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 RIGHT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 0 val_0 0 val_0 +NULL NULL 2 val_2 2 val_2 +NULL NULL 4 val_4 4 val_4 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 5 val_5 5 val_5 +NULL NULL 8 val_8 8 val_8 +NULL NULL 9 val_9 9 val_9 +PREHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + input vertices: + 1 Map 3 + 2 Map 4 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + sort order: ++++++ + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Index: ql/src/test/results/clientpositive/tez/auto_join30.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join30.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/auto_join30.q.out (working copy) @@ -0,0 +1,1335 @@ +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: + 1 Reducer 5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +103231310608 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: + 1 Reducer 5 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +103231310608 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: + 0 Reducer 2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +103231310608 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: + 1 Reducer 5 + 2 Reducer 7 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 7 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +348019368476 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +348019368476 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +348019368476 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +LEFT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +348019368476 +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Right Outer Join0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src sort by key) x +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +RIGHT OUTER JOIN +(SELECT src.* FROM src sort by value) Z +ON (x.key = Z.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +348019368476 Index: ql/src/test/results/clientpositive/tez/auto_join_filters.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join_filters.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/auto_join_filters.q.out (working copy) @@ -0,0 +1,540 @@ +PREHOOK: query: CREATE TABLE myinput1(key int, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@myinput1 +POSTHOOK: query: CREATE TABLE myinput1(key int, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myinput1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@myinput1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@myinput1 +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4937935 +Warning: Map Join MAPJOIN[15][bigTable=b] in task 'Map 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4937935 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4937935 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4937935 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4937935 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3080335 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_input1 +POSTHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_input1 +PREHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_input2 +POSTHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_input2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' into table smb_input1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' into table smb_input1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table smb_input1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table smb_input1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' into table smb_input2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' into table smb_input2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table smb_input2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table smb_input2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input2 +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 Index: ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out (working copy) @@ -0,0 +1,217 @@ +PREHOOK: query: CREATE TABLE myinput1(key int, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@myinput1 +POSTHOOK: query: CREATE TABLE myinput1(key int, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myinput1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@myinput1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@myinput1 +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +13630578 +Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +13630578 +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +13630578 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4509856 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3112070 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4542003 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4542038 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4543491 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4542003 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3079923 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4509891 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3113558 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3079923 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4543526 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4543526 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4543526 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +4543526 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3112070 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3113558 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +3112070 Index: ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out (working copy) @@ -2576,7 +2576,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: '11' (type: string) @@ -2605,7 +2605,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -2715,7 +2715,7 @@ Reduce Output Operator key expressions: '13' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '13' (type: string) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -2729,7 +2729,7 @@ Reduce Output Operator key expressions: '13' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '13' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -4694,7 +4694,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: '11' (type: string) Index: ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out (working copy) @@ -602,7 +602,7 @@ Reduce Output Operator key expressions: 1 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 1 (type: int) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Index: ql/src/test/results/clientpositive/tez/groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/groupby2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/groupby2.q.out (working copy) @@ -91,11 +91,15 @@ POSTHOOK: Lineage: dest_g2.c1 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.c2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_g2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT dest_g2.* FROM dest_g2 +PREHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 PREHOOK: type: QUERY PREHOOK: Input: default@dest_g2 #### A masked pattern was here #### -POSTHOOK: query: SELECT dest_g2.* FROM dest_g2 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +SELECT dest_g2.* FROM dest_g2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_g2 #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/tez/join0.q.java1.7.out =================================================================== --- ql/src/test/results/clientpositive/tez/join0.q.java1.7.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/join0.q.java1.7.out (working copy) @@ -1,5 +1,6 @@ Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, @@ -10,6 +11,7 @@ SORT BY k1, v1, k2, v2 PREHOOK: type: QUERY POSTHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT +-- SORT_QUERY_RESULTS EXPLAIN SELECT src1.key as k1, src1.value as v1, Index: ql/src/test/results/clientpositive/tez/join_nullsafe.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/join_nullsafe.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/join_nullsafe.q.out (working copy) @@ -1578,7 +1578,7 @@ Reduce Output Operator key expressions: null (type: void) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: int) Map 3 @@ -1592,7 +1592,7 @@ Reduce Output Operator key expressions: null (type: void) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: int) Reducer 2 Index: ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out (working copy) @@ -374,119 +374,3 @@ 9 9 9 9 9 9 -PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec) -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -#### A masked pattern was here #### -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -14 14 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -17 17 -45 45 -45 45 -45 45 -45 45 -45 45 -6 6 -6 6 -6 6 -6 6 -6 6 -6 6 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -62 62 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -64 64 -70 70 -70 70 -70 70 -70 70 -70 70 -70 70 -70 70 -79 79 -79 79 -79 79 -79 79 -79 79 -79 79 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -89 89 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 -9 9 Index: ql/src/test/results/clientpositive/tez/scriptfile1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/scriptfile1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/scriptfile1.q.out (working copy) @@ -1,9 +1,13 @@ -PREHOOK: query: -- EXCLUDE_OS_WINDOWS +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: query: -- EXCLUDE_OS_WINDOWS +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- EXCLUDE_OS_WINDOWS CREATE TABLE dest1(key INT, value STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default Index: ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out (revision 0) +++ ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out (working copy) @@ -0,0 +1,4906 @@ +PREHOOK: query: explain select distinct src.* from src +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct src.* from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct src.* from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct src.* from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct * from src where key < '3' +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct * from src where key < '3' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < '3') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct * from src where key < '3' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src where key < '3' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +PREHOOK: query: from src a select distinct a.* where a.key = '238' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: from src a select distinct a.* where a.key = '238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 val_238 +PREHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists sdi +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists sdi +POSTHOOK: type: DROPVIEW +PREHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create View Operator: + Create View + or replace: false + columns: key string, value string + expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 + name: default.sdi + original text: select distinct * from src order by key limit 2 + +PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@sdi +POSTHOOK: query: create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sdi +PREHOOK: query: describe extended sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe extended sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: describe formatted sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe formatted sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select distinct * from src order by key limit 2 +View Expanded Text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 +PREHOOK: query: select * from sdi +PREHOOK: type: QUERY +PREHOOK: Input: default@sdi +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from sdi +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sdi +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +PREHOOK: query: select distinct * from src union all select distinct * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src union all select distinct * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +PREHOOK: query: select distinct * from src join src1 on src.key=src1.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src join src1 on src.key=src1.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +128 128 val_128 +146 val_146 146 val_146 +150 val_150 150 val_150 +213 val_213 213 val_213 +224 224 val_224 +238 val_238 238 val_238 +255 val_255 255 val_255 +273 val_273 273 val_273 +278 val_278 278 val_278 +311 val_311 311 val_311 +369 369 val_369 +401 val_401 401 val_401 +406 val_406 406 val_406 +66 val_66 66 val_66 +98 val_98 98 val_98 +PREHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +146 val_146 2008-04-08 11 146 val_146 146 val_146 +150 val_150 2008-04-08 11 150 val_150 150 val_150 +213 val_213 2008-04-08 11 213 val_213 213 val_213 +238 val_238 2008-04-08 11 238 val_238 238 val_238 +255 val_255 2008-04-08 11 255 val_255 255 val_255 +273 val_273 2008-04-08 11 273 val_273 273 val_273 +278 val_278 2008-04-08 11 278 val_278 278 val_278 +311 val_311 2008-04-08 11 311 val_311 311 val_311 +401 val_401 2008-04-08 11 401 val_401 401 val_401 +406 val_406 2008-04-08 11 406 val_406 406 val_406 +66 val_66 2008-04-08 11 66 val_66 66 val_66 +98 val_98 2008-04-08 11 98 val_98 98 val_98 +PREHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 +103 val_103 103 val_103 +104 val_104 104 val_104 +105 val_105 105 val_105 +11 val_11 11 val_11 +111 val_111 111 val_111 +113 val_113 113 val_113 +114 val_114 114 val_114 +116 val_116 116 val_116 +118 val_118 118 val_118 +119 val_119 119 val_119 +12 val_12 12 val_12 +120 val_120 120 val_120 +125 val_125 125 val_125 +126 val_126 126 val_126 +128 val_128 128 val_128 +129 val_129 129 val_129 +131 val_131 131 val_131 +133 val_133 133 val_133 +134 val_134 134 val_134 +136 val_136 136 val_136 +137 val_137 137 val_137 +138 val_138 138 val_138 +143 val_143 143 val_143 +145 val_145 145 val_145 +146 val_146 146 val_146 +149 val_149 149 val_149 +15 val_15 15 val_15 +150 val_150 150 val_150 +152 val_152 152 val_152 +153 val_153 153 val_153 +155 val_155 155 val_155 +156 val_156 156 val_156 +157 val_157 157 val_157 +158 val_158 158 val_158 +160 val_160 160 val_160 +162 val_162 162 val_162 +163 val_163 163 val_163 +164 val_164 164 val_164 +165 val_165 165 val_165 +166 val_166 166 val_166 +167 val_167 167 val_167 +168 val_168 168 val_168 +169 val_169 169 val_169 +17 val_17 17 val_17 +170 val_170 170 val_170 +172 val_172 172 val_172 +174 val_174 174 val_174 +175 val_175 175 val_175 +176 val_176 176 val_176 +177 val_177 177 val_177 +178 val_178 178 val_178 +179 val_179 179 val_179 +18 val_18 18 val_18 +180 val_180 180 val_180 +181 val_181 181 val_181 +183 val_183 183 val_183 +186 val_186 186 val_186 +187 val_187 187 val_187 +189 val_189 189 val_189 +19 val_19 19 val_19 +190 val_190 190 val_190 +191 val_191 191 val_191 +192 val_192 192 val_192 +193 val_193 193 val_193 +194 val_194 194 val_194 +195 val_195 195 val_195 +196 val_196 196 val_196 +197 val_197 197 val_197 +199 val_199 199 val_199 +2 val_2 2 val_2 +20 val_20 20 val_20 +200 val_200 200 val_200 +201 val_201 201 val_201 +202 val_202 202 val_202 +203 val_203 203 val_203 +205 val_205 205 val_205 +207 val_207 207 val_207 +208 val_208 208 val_208 +209 val_209 209 val_209 +213 val_213 213 val_213 +214 val_214 214 val_214 +216 val_216 216 val_216 +217 val_217 217 val_217 +218 val_218 218 val_218 +219 val_219 219 val_219 +221 val_221 221 val_221 +222 val_222 222 val_222 +223 val_223 223 val_223 +224 val_224 224 val_224 +226 val_226 226 val_226 +228 val_228 228 val_228 +229 val_229 229 val_229 +230 val_230 230 val_230 +233 val_233 233 val_233 +235 val_235 235 val_235 +237 val_237 237 val_237 +238 val_238 238 val_238 +239 val_239 239 val_239 +24 val_24 24 val_24 +241 val_241 241 val_241 +242 val_242 242 val_242 +244 val_244 244 val_244 +247 val_247 247 val_247 +248 val_248 248 val_248 +249 val_249 249 val_249 +252 val_252 252 val_252 +255 val_255 255 val_255 +256 val_256 256 val_256 +257 val_257 257 val_257 +258 val_258 258 val_258 +26 val_26 26 val_26 +260 val_260 260 val_260 +262 val_262 262 val_262 +263 val_263 263 val_263 +265 val_265 265 val_265 +266 val_266 266 val_266 +27 val_27 27 val_27 +272 val_272 272 val_272 +273 val_273 273 val_273 +274 val_274 274 val_274 +275 val_275 275 val_275 +277 val_277 277 val_277 +278 val_278 278 val_278 +28 val_28 28 val_28 +280 val_280 280 val_280 +281 val_281 281 val_281 +282 val_282 282 val_282 +283 val_283 283 val_283 +284 val_284 284 val_284 +285 val_285 285 val_285 +286 val_286 286 val_286 +287 val_287 287 val_287 +288 val_288 288 val_288 +289 val_289 289 val_289 +291 val_291 291 val_291 +292 val_292 292 val_292 +296 val_296 296 val_296 +298 val_298 298 val_298 +30 val_30 30 val_30 +302 val_302 302 val_302 +305 val_305 305 val_305 +306 val_306 306 val_306 +307 val_307 307 val_307 +308 val_308 308 val_308 +309 val_309 309 val_309 +310 val_310 310 val_310 +311 val_311 311 val_311 +315 val_315 315 val_315 +316 val_316 316 val_316 +317 val_317 317 val_317 +318 val_318 318 val_318 +321 val_321 321 val_321 +322 val_322 322 val_322 +323 val_323 323 val_323 +325 val_325 325 val_325 +327 val_327 327 val_327 +33 val_33 33 val_33 +331 val_331 331 val_331 +332 val_332 332 val_332 +333 val_333 333 val_333 +335 val_335 335 val_335 +336 val_336 336 val_336 +338 val_338 338 val_338 +339 val_339 339 val_339 +34 val_34 34 val_34 +341 val_341 341 val_341 +342 val_342 342 val_342 +344 val_344 344 val_344 +345 val_345 345 val_345 +348 val_348 348 val_348 +35 val_35 35 val_35 +351 val_351 351 val_351 +353 val_353 353 val_353 +356 val_356 356 val_356 +360 val_360 360 val_360 +362 val_362 362 val_362 +364 val_364 364 val_364 +365 val_365 365 val_365 +366 val_366 366 val_366 +367 val_367 367 val_367 +368 val_368 368 val_368 +369 val_369 369 val_369 +37 val_37 37 val_37 +373 val_373 373 val_373 +374 val_374 374 val_374 +375 val_375 375 val_375 +377 val_377 377 val_377 +378 val_378 378 val_378 +379 val_379 379 val_379 +382 val_382 382 val_382 +384 val_384 384 val_384 +386 val_386 386 val_386 +389 val_389 389 val_389 +392 val_392 392 val_392 +393 val_393 393 val_393 +394 val_394 394 val_394 +395 val_395 395 val_395 +396 val_396 396 val_396 +397 val_397 397 val_397 +399 val_399 399 val_399 +4 val_4 4 val_4 +400 val_400 400 val_400 +401 val_401 401 val_401 +402 val_402 402 val_402 +403 val_403 403 val_403 +404 val_404 404 val_404 +406 val_406 406 val_406 +407 val_407 407 val_407 +409 val_409 409 val_409 +41 val_41 41 val_41 +411 val_411 411 val_411 +413 val_413 413 val_413 +414 val_414 414 val_414 +417 val_417 417 val_417 +418 val_418 418 val_418 +419 val_419 419 val_419 +42 val_42 42 val_42 +421 val_421 421 val_421 +424 val_424 424 val_424 +427 val_427 427 val_427 +429 val_429 429 val_429 +43 val_43 43 val_43 +430 val_430 430 val_430 +431 val_431 431 val_431 +432 val_432 432 val_432 +435 val_435 435 val_435 +436 val_436 436 val_436 +437 val_437 437 val_437 +438 val_438 438 val_438 +439 val_439 439 val_439 +44 val_44 44 val_44 +443 val_443 443 val_443 +444 val_444 444 val_444 +446 val_446 446 val_446 +448 val_448 448 val_448 +449 val_449 449 val_449 +452 val_452 452 val_452 +453 val_453 453 val_453 +454 val_454 454 val_454 +455 val_455 455 val_455 +457 val_457 457 val_457 +458 val_458 458 val_458 +459 val_459 459 val_459 +460 val_460 460 val_460 +462 val_462 462 val_462 +463 val_463 463 val_463 +466 val_466 466 val_466 +467 val_467 467 val_467 +468 val_468 468 val_468 +469 val_469 469 val_469 +47 val_47 47 val_47 +470 val_470 470 val_470 +472 val_472 472 val_472 +475 val_475 475 val_475 +477 val_477 477 val_477 +478 val_478 478 val_478 +479 val_479 479 val_479 +480 val_480 480 val_480 +481 val_481 481 val_481 +482 val_482 482 val_482 +483 val_483 483 val_483 +484 val_484 484 val_484 +485 val_485 485 val_485 +487 val_487 487 val_487 +489 val_489 489 val_489 +490 val_490 490 val_490 +491 val_491 491 val_491 +492 val_492 492 val_492 +493 val_493 493 val_493 +494 val_494 494 val_494 +495 val_495 495 val_495 +496 val_496 496 val_496 +497 val_497 497 val_497 +498 val_498 498 val_498 +5 val_5 5 val_5 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +8 val_8 8 val_8 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +9 val_9 9 val_9 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +98 val_98 98 val_98 +PREHOOK: query: select distinct * from (select distinct * from src)src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from (select distinct * from src)src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct src.* from src +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct src.* from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct src.* from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct src.* from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: explain select distinct * from src where key < '3' +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct * from src where key < '3' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < '3') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select distinct * from src where key < '3' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src where key < '3' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +PREHOOK: query: from src a select distinct a.* where a.key = '238' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: from src a select distinct a.* where a.key = '238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 val_238 +PREHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 2746 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * from ( +select * from src1 +union all +select * from src )subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists sdi +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@sdi +PREHOOK: Output: default@sdi +POSTHOOK: query: drop view if exists sdi +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@sdi +POSTHOOK: Output: default@sdi +PREHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create View Operator: + Create View + or replace: false + columns: key string, value string + expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 + name: default.sdi + original text: select distinct * from src order by key limit 2 + +PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@sdi +POSTHOOK: query: create view sdi as select distinct * from src order by key limit 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sdi +PREHOOK: query: describe extended sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe extended sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: describe formatted sdi +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sdi +POSTHOOK: query: describe formatted sdi +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sdi +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select distinct * from src order by key limit 2 +View Expanded Text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 +PREHOOK: query: select * from sdi +PREHOOK: type: QUERY +PREHOOK: Input: default@sdi +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from sdi +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sdi +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +PREHOOK: query: select distinct * from src union all select distinct * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src union all select distinct * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +PREHOOK: query: select distinct * from src join src1 on src.key=src1.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from src join src1 on src.key=src1.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +128 128 val_128 +146 val_146 146 val_146 +150 val_150 150 val_150 +213 val_213 213 val_213 +224 224 val_224 +238 val_238 238 val_238 +255 val_255 255 val_255 +273 val_273 273 val_273 +278 val_278 278 val_278 +311 val_311 311 val_311 +369 369 val_369 +401 val_401 401 val_401 +406 val_406 406 val_406 +66 val_66 66 val_66 +98 val_98 98 val_98 +PREHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct * +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +146 val_146 2008-04-08 11 146 val_146 146 val_146 +150 val_150 2008-04-08 11 150 val_150 150 val_150 +213 val_213 2008-04-08 11 213 val_213 213 val_213 +238 val_238 2008-04-08 11 238 val_238 238 val_238 +255 val_255 2008-04-08 11 255 val_255 255 val_255 +273 val_273 2008-04-08 11 273 val_273 273 val_273 +278 val_278 2008-04-08 11 278 val_278 278 val_278 +311 val_311 2008-04-08 11 311 val_311 311 val_311 +401 val_401 2008-04-08 11 401 val_401 401 val_401 +406 val_406 2008-04-08 11 406 val_406 406 val_406 +66 val_66 2008-04-08 11 66 val_66 66 val_66 +98 val_98 2008-04-08 11 98 val_98 98 val_98 +PREHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select distinct * from src)src1 +join +(select distinct * from src)src2 +on src1.key=src2.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 +103 val_103 103 val_103 +104 val_104 104 val_104 +105 val_105 105 val_105 +11 val_11 11 val_11 +111 val_111 111 val_111 +113 val_113 113 val_113 +114 val_114 114 val_114 +116 val_116 116 val_116 +118 val_118 118 val_118 +119 val_119 119 val_119 +12 val_12 12 val_12 +120 val_120 120 val_120 +125 val_125 125 val_125 +126 val_126 126 val_126 +128 val_128 128 val_128 +129 val_129 129 val_129 +131 val_131 131 val_131 +133 val_133 133 val_133 +134 val_134 134 val_134 +136 val_136 136 val_136 +137 val_137 137 val_137 +138 val_138 138 val_138 +143 val_143 143 val_143 +145 val_145 145 val_145 +146 val_146 146 val_146 +149 val_149 149 val_149 +15 val_15 15 val_15 +150 val_150 150 val_150 +152 val_152 152 val_152 +153 val_153 153 val_153 +155 val_155 155 val_155 +156 val_156 156 val_156 +157 val_157 157 val_157 +158 val_158 158 val_158 +160 val_160 160 val_160 +162 val_162 162 val_162 +163 val_163 163 val_163 +164 val_164 164 val_164 +165 val_165 165 val_165 +166 val_166 166 val_166 +167 val_167 167 val_167 +168 val_168 168 val_168 +169 val_169 169 val_169 +17 val_17 17 val_17 +170 val_170 170 val_170 +172 val_172 172 val_172 +174 val_174 174 val_174 +175 val_175 175 val_175 +176 val_176 176 val_176 +177 val_177 177 val_177 +178 val_178 178 val_178 +179 val_179 179 val_179 +18 val_18 18 val_18 +180 val_180 180 val_180 +181 val_181 181 val_181 +183 val_183 183 val_183 +186 val_186 186 val_186 +187 val_187 187 val_187 +189 val_189 189 val_189 +19 val_19 19 val_19 +190 val_190 190 val_190 +191 val_191 191 val_191 +192 val_192 192 val_192 +193 val_193 193 val_193 +194 val_194 194 val_194 +195 val_195 195 val_195 +196 val_196 196 val_196 +197 val_197 197 val_197 +199 val_199 199 val_199 +2 val_2 2 val_2 +20 val_20 20 val_20 +200 val_200 200 val_200 +201 val_201 201 val_201 +202 val_202 202 val_202 +203 val_203 203 val_203 +205 val_205 205 val_205 +207 val_207 207 val_207 +208 val_208 208 val_208 +209 val_209 209 val_209 +213 val_213 213 val_213 +214 val_214 214 val_214 +216 val_216 216 val_216 +217 val_217 217 val_217 +218 val_218 218 val_218 +219 val_219 219 val_219 +221 val_221 221 val_221 +222 val_222 222 val_222 +223 val_223 223 val_223 +224 val_224 224 val_224 +226 val_226 226 val_226 +228 val_228 228 val_228 +229 val_229 229 val_229 +230 val_230 230 val_230 +233 val_233 233 val_233 +235 val_235 235 val_235 +237 val_237 237 val_237 +238 val_238 238 val_238 +239 val_239 239 val_239 +24 val_24 24 val_24 +241 val_241 241 val_241 +242 val_242 242 val_242 +244 val_244 244 val_244 +247 val_247 247 val_247 +248 val_248 248 val_248 +249 val_249 249 val_249 +252 val_252 252 val_252 +255 val_255 255 val_255 +256 val_256 256 val_256 +257 val_257 257 val_257 +258 val_258 258 val_258 +26 val_26 26 val_26 +260 val_260 260 val_260 +262 val_262 262 val_262 +263 val_263 263 val_263 +265 val_265 265 val_265 +266 val_266 266 val_266 +27 val_27 27 val_27 +272 val_272 272 val_272 +273 val_273 273 val_273 +274 val_274 274 val_274 +275 val_275 275 val_275 +277 val_277 277 val_277 +278 val_278 278 val_278 +28 val_28 28 val_28 +280 val_280 280 val_280 +281 val_281 281 val_281 +282 val_282 282 val_282 +283 val_283 283 val_283 +284 val_284 284 val_284 +285 val_285 285 val_285 +286 val_286 286 val_286 +287 val_287 287 val_287 +288 val_288 288 val_288 +289 val_289 289 val_289 +291 val_291 291 val_291 +292 val_292 292 val_292 +296 val_296 296 val_296 +298 val_298 298 val_298 +30 val_30 30 val_30 +302 val_302 302 val_302 +305 val_305 305 val_305 +306 val_306 306 val_306 +307 val_307 307 val_307 +308 val_308 308 val_308 +309 val_309 309 val_309 +310 val_310 310 val_310 +311 val_311 311 val_311 +315 val_315 315 val_315 +316 val_316 316 val_316 +317 val_317 317 val_317 +318 val_318 318 val_318 +321 val_321 321 val_321 +322 val_322 322 val_322 +323 val_323 323 val_323 +325 val_325 325 val_325 +327 val_327 327 val_327 +33 val_33 33 val_33 +331 val_331 331 val_331 +332 val_332 332 val_332 +333 val_333 333 val_333 +335 val_335 335 val_335 +336 val_336 336 val_336 +338 val_338 338 val_338 +339 val_339 339 val_339 +34 val_34 34 val_34 +341 val_341 341 val_341 +342 val_342 342 val_342 +344 val_344 344 val_344 +345 val_345 345 val_345 +348 val_348 348 val_348 +35 val_35 35 val_35 +351 val_351 351 val_351 +353 val_353 353 val_353 +356 val_356 356 val_356 +360 val_360 360 val_360 +362 val_362 362 val_362 +364 val_364 364 val_364 +365 val_365 365 val_365 +366 val_366 366 val_366 +367 val_367 367 val_367 +368 val_368 368 val_368 +369 val_369 369 val_369 +37 val_37 37 val_37 +373 val_373 373 val_373 +374 val_374 374 val_374 +375 val_375 375 val_375 +377 val_377 377 val_377 +378 val_378 378 val_378 +379 val_379 379 val_379 +382 val_382 382 val_382 +384 val_384 384 val_384 +386 val_386 386 val_386 +389 val_389 389 val_389 +392 val_392 392 val_392 +393 val_393 393 val_393 +394 val_394 394 val_394 +395 val_395 395 val_395 +396 val_396 396 val_396 +397 val_397 397 val_397 +399 val_399 399 val_399 +4 val_4 4 val_4 +400 val_400 400 val_400 +401 val_401 401 val_401 +402 val_402 402 val_402 +403 val_403 403 val_403 +404 val_404 404 val_404 +406 val_406 406 val_406 +407 val_407 407 val_407 +409 val_409 409 val_409 +41 val_41 41 val_41 +411 val_411 411 val_411 +413 val_413 413 val_413 +414 val_414 414 val_414 +417 val_417 417 val_417 +418 val_418 418 val_418 +419 val_419 419 val_419 +42 val_42 42 val_42 +421 val_421 421 val_421 +424 val_424 424 val_424 +427 val_427 427 val_427 +429 val_429 429 val_429 +43 val_43 43 val_43 +430 val_430 430 val_430 +431 val_431 431 val_431 +432 val_432 432 val_432 +435 val_435 435 val_435 +436 val_436 436 val_436 +437 val_437 437 val_437 +438 val_438 438 val_438 +439 val_439 439 val_439 +44 val_44 44 val_44 +443 val_443 443 val_443 +444 val_444 444 val_444 +446 val_446 446 val_446 +448 val_448 448 val_448 +449 val_449 449 val_449 +452 val_452 452 val_452 +453 val_453 453 val_453 +454 val_454 454 val_454 +455 val_455 455 val_455 +457 val_457 457 val_457 +458 val_458 458 val_458 +459 val_459 459 val_459 +460 val_460 460 val_460 +462 val_462 462 val_462 +463 val_463 463 val_463 +466 val_466 466 val_466 +467 val_467 467 val_467 +468 val_468 468 val_468 +469 val_469 469 val_469 +47 val_47 47 val_47 +470 val_470 470 val_470 +472 val_472 472 val_472 +475 val_475 475 val_475 +477 val_477 477 val_477 +478 val_478 478 val_478 +479 val_479 479 val_479 +480 val_480 480 val_480 +481 val_481 481 val_481 +482 val_482 482 val_482 +483 val_483 483 val_483 +484 val_484 484 val_484 +485 val_485 485 val_485 +487 val_487 487 val_487 +489 val_489 489 val_489 +490 val_490 490 val_490 +491 val_491 491 val_491 +492 val_492 492 val_492 +493 val_493 493 val_493 +494 val_494 494 val_494 +495 val_495 495 val_495 +496 val_496 496 val_496 +497 val_497 497 val_497 +498 val_498 498 val_498 +5 val_5 5 val_5 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +8 val_8 8 val_8 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +9 val_9 9 val_9 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +98 val_98 98 val_98 +PREHOOK: query: select distinct * from (select distinct * from src)src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from (select distinct * from src)src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 Index: ql/src/test/results/clientpositive/tez/subquery_in.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/subquery_in.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/subquery_in.q.out (working copy) @@ -310,7 +310,6 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -334,7 +333,9 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -466,7 +467,6 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -490,7 +490,9 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/tez/transform_ppr1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/transform_ppr1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/transform_ppr1.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -7,7 +9,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/tez/transform_ppr2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/transform_ppr2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/transform_ppr2.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -8,7 +10,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/tez/union6.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/union6.q.out (working copy) @@ -1,12 +1,14 @@ -PREHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE Index: ql/src/test/results/clientpositive/tez/vector_decimal_mapjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/vector_decimal_mapjoin.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/vector_decimal_mapjoin.q.out (working copy) @@ -82,7 +82,7 @@ Reduce Output Operator key expressions: 6981 (type: int) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: 6981 (type: int) Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE value expressions: cdecimal2 (type: decimal(23,14)) Execution mode: vectorized Index: ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out (working copy) @@ -200,14 +200,14 @@ predicate: (((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: Index: ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out (working copy) @@ -2606,7 +2606,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: '11' (type: string) @@ -2635,7 +2635,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -2747,7 +2747,7 @@ Reduce Output Operator key expressions: '13' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '13' (type: string) Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -2761,7 +2761,7 @@ Reduce Output Operator key expressions: '13' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '13' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -4748,7 +4748,7 @@ Reduce Output Operator key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '11' (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: '11' (type: string) Index: ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out (working copy) @@ -312,12 +312,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -690,12 +692,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -1172,12 +1176,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1460,12 +1466,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1778,13 +1786,15 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col2 (type: int) auto parallelism: true Execution mode: vectorized Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -2661,12 +2671,13 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2938,12 +2949,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3217,12 +3230,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3523,12 +3538,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3832,12 +3849,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4309,12 +4328,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -4842,12 +4863,14 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col2 (type: double) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -5284,7 +5307,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string) auto parallelism: true Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) @@ -5296,12 +5319,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5311,12 +5336,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _wcol0 (type: bigint), _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _wcol0 (type: bigint), _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: bigint), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5353,7 +5380,9 @@ Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5813,12 +5842,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6183,12 +6214,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6521,12 +6554,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6888,12 +6923,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 5 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7271,12 +7308,14 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7620,12 +7659,14 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/transform_ppr1.q.out =================================================================== --- ql/src/test/results/clientpositive/transform_ppr1.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/transform_ppr1.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -7,7 +9,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/transform_ppr2.q.out =================================================================== --- ql/src/test/results/clientpositive/transform_ppr2.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/transform_ppr2.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -8,7 +10,9 @@ ) tmap SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) Index: ql/src/test/results/clientpositive/udf_add_months.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_add_months.q.out (revision 0) +++ ql/src/test/results/clientpositive/udf_add_months.q.out (working copy) @@ -0,0 +1,14 @@ +PREHOOK: query: DESCRIBE FUNCTION add_months +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION add_months +POSTHOOK: type: DESCFUNCTION +add_months(start_date, num_months) - Returns the date that is num_months after start_date. +PREHOOK: query: DESCRIBE FUNCTION EXTENDED add_months +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED add_months +POSTHOOK: type: DESCFUNCTION +add_months(start_date, num_months) - Returns the date that is num_months after start_date. +start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_months is a number. The time part of start_date is ignored. +Example: + > SELECT add_months('2009-08-31', 1) FROM src LIMIT 1; + '2009-09-30' Index: ql/src/test/results/clientpositive/udf_date_add.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_date_add.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/udf_date_add.q.out (working copy) @@ -10,5 +10,5 @@ date_add(start_date, num_days) - Returns the date that is num_days after start_date. start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_days is a number. The time part of start_date is ignored. Example: - > SELECT date_add('2009-30-07', 1) FROM src LIMIT 1; - '2009-31-07' + > SELECT date_add('2009-07-30', 1) FROM src LIMIT 1; + '2009-07-31' Index: ql/src/test/results/clientpositive/udf_date_sub.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_date_sub.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/udf_date_sub.q.out (working copy) @@ -10,5 +10,5 @@ date_sub(start_date, num_days) - Returns the date that is num_days before start_date. start_date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. num_days is a number. The time part of start_date is ignored. Example: - > SELECT date_sub('2009-30-07', 1) FROM src LIMIT 1; - '2009-29-07' + > SELECT date_sub('2009-07-30', 1) FROM src LIMIT 1; + '2009-07-29' Index: ql/src/test/results/clientpositive/udf_datediff.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_datediff.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/udf_datediff.q.out (working copy) @@ -10,5 +10,5 @@ datediff(date1, date2) - Returns the number of days between date1 and date2 date1 and date2 are strings in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time parts are ignored.If date1 is earlier than date2, the result is negative. Example: - > SELECT datediff('2009-30-07', '2009-31-07') FROM src LIMIT 1; + > SELECT datediff('2009-07-30', '2009-07-31') FROM src LIMIT 1; 1 Index: ql/src/test/results/clientpositive/udf_day.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_day.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/udf_day.q.out (working copy) @@ -11,5 +11,5 @@ Synonyms: dayofmonth date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. Example: - > SELECT day('2009-30-07', 1) FROM src LIMIT 1; + > SELECT day('2009-07-30') FROM src LIMIT 1; 30 Index: ql/src/test/results/clientpositive/udf_dayofmonth.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_dayofmonth.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/udf_dayofmonth.q.out (working copy) @@ -11,5 +11,5 @@ Synonyms: day date is a string in the format of 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. Example: - > SELECT dayofmonth('2009-30-07', 1) FROM src LIMIT 1; + > SELECT dayofmonth('2009-07-30') FROM src LIMIT 1; 30 Index: ql/src/test/results/clientpositive/udf_initcap.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_initcap.q.out (revision 0) +++ ql/src/test/results/clientpositive/udf_initcap.q.out (working copy) @@ -0,0 +1,13 @@ +PREHOOK: query: DESCRIBE FUNCTION initcap +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION initcap +POSTHOOK: type: DESCFUNCTION +initcap(str) - Returns str, with the first letter of each word in uppercase, all other letters in lowercase. Words are delimited by white space. +PREHOOK: query: DESCRIBE FUNCTION EXTENDED initcap +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED initcap +POSTHOOK: type: DESCFUNCTION +initcap(str) - Returns str, with the first letter of each word in uppercase, all other letters in lowercase. Words are delimited by white space. +Example: + > SELECT initcap('tHe soap') FROM src LIMIT 1; + 'The Soap' Index: ql/src/test/results/clientpositive/udf_last_day.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_last_day.q.out (revision 0) +++ ql/src/test/results/clientpositive/udf_last_day.q.out (working copy) @@ -0,0 +1,14 @@ +PREHOOK: query: DESCRIBE FUNCTION last_day +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION last_day +POSTHOOK: type: DESCFUNCTION +last_day(date) - Returns the last day of the month which the date belongs to. +PREHOOK: query: DESCRIBE FUNCTION EXTENDED last_day +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED last_day +POSTHOOK: type: DESCFUNCTION +last_day(date) - Returns the last day of the month which the date belongs to. +date is a string in the format 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd'. The time part of date is ignored. +Example: + > SELECT last_day('2009-01-12') FROM src LIMIT 1; + '2009-01-31' Index: ql/src/test/results/clientpositive/union10.q.out =================================================================== --- ql/src/test/results/clientpositive/union10.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union10.q.out (working copy) @@ -1,11 +1,15 @@ -PREHOOK: query: -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + create table tmptable(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + create table tmptable(key string, value int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default Index: ql/src/test/results/clientpositive/union18.q.out =================================================================== --- ql/src/test/results/clientpositive/union18.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union18.q.out (working copy) @@ -14,8 +14,10 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST2 -PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case:map-reduce sub-queries followed by multi-table insert + explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL @@ -23,8 +25,10 @@ INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY -POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case:map-reduce sub-queries followed by multi-table insert + explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL Index: ql/src/test/results/clientpositive/union19.q.out =================================================================== --- ql/src/test/results/clientpositive/union19.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union19.q.out (working copy) @@ -1,8 +1,12 @@ -PREHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST1 -POSTHOOK: query: CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST1 Index: ql/src/test/results/clientpositive/union27.q.out =================================================================== --- ql/src/test/results/clientpositive/union27.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union27.q.out (working copy) @@ -57,7 +57,7 @@ Reduce Output Operator key expressions: '97' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '97' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan @@ -75,7 +75,7 @@ Reduce Output Operator key expressions: '97' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '97' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan @@ -87,7 +87,7 @@ Reduce Output Operator key expressions: '97' (type: string) sort order: + - Map-reduce partition columns: '' (type: string) + Map-reduce partition columns: '97' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Index: ql/src/test/results/clientpositive/union6.q.out =================================================================== --- ql/src/test/results/clientpositive/union6.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union6.q.out (working copy) @@ -1,12 +1,14 @@ -PREHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +PREHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable -POSTHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink +POSTHOOK: query: -- SORT_QUERY_RESULTS +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE Index: ql/src/test/results/clientpositive/union_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/union_ppr.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union_ppr.q.out (working copy) @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED SELECT * FROM ( SELECT X.* FROM SRCPART X WHERE X.key < 100 UNION ALL @@ -7,7 +9,9 @@ WHERE A.ds = '2008-04-08' SORT BY A.key, A.value, A.ds, A.hr PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED SELECT * FROM ( SELECT X.* FROM SRCPART X WHERE X.key < 100 UNION ALL Index: ql/src/test/results/clientpositive/union_remove_6_subq.q.out =================================================================== --- ql/src/test/results/clientpositive/union_remove_6_subq.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/union_remove_6_subq.q.out (working copy) @@ -544,9 +544,10 @@ sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out =================================================================== --- ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out (working copy) @@ -475,7 +475,7 @@ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) 1 _col0 (type: int), _col1 (type: int) Stage: Stage-8 @@ -488,14 +488,14 @@ predicate: (((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 Statistics: Num rows: 14 Data size: 1714 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/vectorized_ptf.q.out =================================================================== --- ql/src/test/results/clientpositive/vectorized_ptf.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/vectorized_ptf.q.out (working copy) @@ -329,7 +329,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -358,7 +358,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -742,7 +744,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -771,7 +773,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -1259,7 +1263,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1288,7 +1292,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1588,7 +1594,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1617,7 +1623,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1994,7 +2002,7 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + value expressions: _col2 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2023,7 +2031,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -2983,7 +2993,6 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3012,7 +3021,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3301,7 +3312,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3330,7 +3341,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3621,7 +3634,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3650,7 +3663,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4015,7 +4030,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4044,7 +4059,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4365,7 +4382,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4394,7 +4411,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4922,7 +4941,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4951,7 +4970,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -5582,7 +5603,7 @@ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col2 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -5611,7 +5632,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -6085,7 +6108,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6114,7 +6137,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6147,7 +6172,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _wcol0 (type: bigint), _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _wcol0 (type: bigint), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6176,7 +6201,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: bigint), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6248,7 +6275,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6277,7 +6304,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6772,7 +6801,7 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6801,7 +6830,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7277,7 +7308,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7306,7 +7337,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7703,7 +7736,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7732,7 +7765,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8205,7 +8240,7 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8234,7 +8269,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8676,7 +8713,7 @@ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8705,7 +8742,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -9113,7 +9152,7 @@ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -9142,7 +9181,9 @@ #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/windowing_streaming.q.out =================================================================== --- ql/src/test/results/clientpositive/windowing_streaming.q.out (revision 1653459) +++ ql/src/test/results/clientpositive/windowing_streaming.q.out (working copy) @@ -74,9 +74,10 @@ sort order: ++ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -129,9 +130,10 @@ Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.8 - value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -282,9 +284,10 @@ Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.8 - value expressions: _col0 (type: tinyint), _col1 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1 Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/templates/TestCliDriver.vm =================================================================== --- ql/src/test/templates/TestCliDriver.vm (revision 1653459) +++ ql/src/test/templates/TestCliDriver.vm (working copy) @@ -52,7 +52,7 @@ System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception in static initialization"+e.getMessage()); + fail("Unexpected exception in static initialization: "+e.getMessage()); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (revision 1653459) +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (working copy) @@ -113,7 +113,7 @@ cachedObjectInspector, notSkipIDs, serdeParams.getNullSequence()); super.initialize(size); - LOG.info("ColumnarSerDe initialized with: columnNames=" + LOG.debug("ColumnarSerDe initialized with: columnNames=" + serdeParams.getColumnNames() + " columnTypes=" + serdeParams.getColumnTypes() + " separator=" + Arrays.asList(serdeParams.getSeparators()) + " nullstring=" Index: service/src/java/org/apache/hive/service/cli/session/HiveSession.java =================================================================== --- service/src/java/org/apache/hive/service/cli/session/HiveSession.java (revision 1653459) +++ service/src/java/org/apache/hive/service/cli/session/HiveSession.java (working copy) @@ -27,7 +27,7 @@ public interface HiveSession extends HiveSessionBase { - void open(); + void open(Map sessionConfMap) throws Exception; IMetaStoreClient getMetaStoreClient() throws HiveSQLException; Index: service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java =================================================================== --- service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java (revision 1653459) +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java (working copy) @@ -55,12 +55,6 @@ void setOperationManager(OperationManager operationManager); /** - * Initialize the session - * @param sessionConfMap - */ - void initialize(Map sessionConfMap) throws Exception; - - /** * Check whether operation logging is enabled and session dir is created successfully */ boolean isOperationLogEnabled(); Index: service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java =================================================================== --- service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (revision 1653459) +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (working copy) @@ -47,7 +47,15 @@ import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.service.auth.HiveAuthFactory; -import org.apache.hive.service.cli.*; +import org.apache.hive.service.cli.FetchOrientation; +import org.apache.hive.service.cli.FetchType; +import org.apache.hive.service.cli.GetInfoType; +import org.apache.hive.service.cli.GetInfoValue; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationHandle; +import org.apache.hive.service.cli.RowSet; +import org.apache.hive.service.cli.SessionHandle; +import org.apache.hive.service.cli.TableSchema; import org.apache.hive.service.cli.operation.ExecuteStatementOperation; import org.apache.hive.service.cli.operation.GetCatalogsOperation; import org.apache.hive.service.cli.operation.GetColumnsOperation; @@ -66,26 +74,20 @@ * */ public class HiveSessionImpl implements HiveSession { - private final SessionHandle sessionHandle; - private String username; private final String password; private HiveConf hiveConf; - private final SessionState sessionState; + private SessionState sessionState; private String ipAddress; - private static final String FETCH_WORK_SERDE_CLASS = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; private static final Log LOG = LogFactory.getLog(HiveSessionImpl.class); - - private SessionManager sessionManager; private OperationManager operationManager; private final Set opHandleSet = new HashSet(); private boolean isOperationLogEnabled; private File sessionLogDir; - private volatile long lastAccessTime; public HiveSessionImpl(TProtocolVersion protocol, String username, String password, @@ -106,7 +108,6 @@ } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } - // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); @@ -114,37 +115,36 @@ hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER, FetchFormatter.ThriftFormatter.class.getName()); hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); + } - /** - * Create a new SessionState object that will be associated with this HiveServer2 session. - * When the server executes multiple queries in the same session, - * this SessionState object is reused across multiple queries. - */ + @Override + /** + * Opens a new HiveServer2 session for the client connection. + * Creates a new SessionState object that will be associated with this HiveServer2 session. + * When the server executes multiple queries in the same session, + * this SessionState object is reused across multiple queries. + * Note that if doAs is true, this call goes through a proxy object, + * which wraps the method logic in a UserGroupInformation#doAs. + * That's why it is important to create SessionState here rather than in the constructor. + */ + public void open(Map sessionConfMap) throws HiveSQLException { sessionState = new SessionState(hiveConf, username); sessionState.setUserIpAddress(ipAddress); sessionState.setIsHiveServerQuery(true); - - lastAccessTime = System.currentTimeMillis(); SessionState.start(sessionState); - } - - @Override - public void initialize(Map sessionConfMap) throws Exception { - // Process global init file: .hiverc - processGlobalInitFile(); try { sessionState.reloadAuxJars(); } catch (IOException e) { - String msg = "fail to load reloadable jar file path" + e; + String msg = "Failed to load reloadable jar file path: " + e; LOG.error(msg, e); - throw new Exception(msg, e); + throw new HiveSQLException(msg, e); } - SessionState.setCurrentSessionState(sessionState); - - // Set conf properties specified by user from client side + // Process global init file: .hiverc + processGlobalInitFile(); if (sessionConfMap != null) { configureSession(sessionConfMap); } + lastAccessTime = System.currentTimeMillis(); } /** @@ -199,12 +199,16 @@ } } - private void configureSession(Map sessionConfMap) throws Exception { + private void configureSession(Map sessionConfMap) throws HiveSQLException { SessionState.setCurrentSessionState(sessionState); for (Map.Entry entry : sessionConfMap.entrySet()) { String key = entry.getKey(); if (key.startsWith("set:")) { - SetProcessor.setVariable(key.substring(4), entry.getValue()); + try { + SetProcessor.setVariable(key.substring(4), entry.getValue()); + } catch (Exception e) { + throw new HiveSQLException(e); + } } else if (key.startsWith("use:")) { SessionState.get().setCurrentDatabase(entry.getValue()); } else { @@ -217,7 +221,6 @@ public void setOperationLogSessionDir(File operationLogRootDir) { sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; - if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + @@ -225,7 +228,6 @@ isOperationLogEnabled = false; } } - if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } @@ -265,19 +267,8 @@ this.operationManager = operationManager; } - @Override - /** - * Opens a new HiveServer2 session for the client connection. - * Note that if doAs is true, this call goes through a proxy object, - * which wraps the method logic in a UserGroupInformation#doAs. - * That is why it is important to call SessionState#start here rather than the constructor. - */ - public void open() { - SessionState.start(sessionState); - } - protected synchronized void acquire(boolean userAccess) { - // Need to make sure that the this HiveServer2's session's session state is + // Need to make sure that the this HiveServer2's session's SessionState is // stored in the thread local for the handler thread. SessionState.setCurrentSessionState(sessionState); if (userAccess) { @@ -556,7 +547,6 @@ opHandleSet.clear(); // Cleanup session log directory. cleanupSessionLogDir(); - HiveHistory hiveHist = sessionState.getHiveHistory(); if (null != hiveHist) { hiveHist.closeStream(); Index: service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java =================================================================== --- service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (revision 1653459) +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (working copy) @@ -97,20 +97,25 @@ } /** - * close the file systems for the session - * cancel the session's delegation token and close the metastore connection + * Close the file systems for the session and remove it from the FileSystem cache. + * Cancel the session's delegation token and close the metastore connection */ @Override public void close() throws HiveSQLException { try { - acquire(true); - FileSystem.closeAllForUGI(sessionUgi); - cancelDelegationToken(); - } catch (IOException ioe) { - LOG.error("Could not clean up file-system handles for UGI: " + sessionUgi, ioe); + acquire(true); + cancelDelegationToken(); } finally { - release(true); - super.close(); + try { + super.close(); + } finally { + try { + FileSystem.closeAllForUGI(sessionUgi); + } catch (IOException ioe) { + throw new HiveSQLException("Could not clean up file-system handles for UGI: " + + sessionUgi, ioe); + } + } } } Index: service/src/java/org/apache/hive/service/cli/session/SessionManager.java =================================================================== --- service/src/java/org/apache/hive/service/cli/session/SessionManager.java (revision 1653459) +++ service/src/java/org/apache/hive/service/cli/session/SessionManager.java (working copy) @@ -267,22 +267,19 @@ session.setSessionManager(this); session.setOperationManager(operationManager); try { - session.initialize(sessionConf); - if (isOperationLogEnabled) { - session.setOperationLogSessionDir(operationLogRootDir); - } - session.open(); + session.open(sessionConf); } catch (Exception e) { - throw new HiveSQLException("Failed to open new session", e); + throw new HiveSQLException("Failed to open new session: " + e, e); } + if (isOperationLogEnabled) { + session.setOperationLogSessionDir(operationLogRootDir); + } try { executeSessionHooks(session); } catch (Exception e) { throw new HiveSQLException("Failed to execute session hooks", e); } - handleToSession.put(session.getSessionHandle(), session); - return session.getSessionHandle(); } Index: shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java =================================================================== --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (revision 1653459) +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (working copy) @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.shims; import java.io.IOException; +import java.lang.Override; +import java.lang.reflect.Constructor; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URI; @@ -63,6 +65,7 @@ import org.apache.hadoop.security.KerberosName; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.VersionInfo; @@ -606,4 +609,39 @@ return kerberosName.getShortName(); } } + + @Override + public StoragePolicyShim getStoragePolicyShim(FileSystem fs) { + return null; + } + + @Override + public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException { + int rc; + + // Creates the command-line parameters for distcp + String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()}; + + try { + Class clazzDistCp = Class.forName("org.apache.hadoop.tools.distcp2"); + Constructor c = clazzDistCp.getConstructor(); + c.setAccessible(true); + Tool distcp = (Tool)c.newInstance(); + distcp.setConf(conf); + rc = distcp.run(params); + } catch (ClassNotFoundException e) { + throw new IOException("Cannot find DistCp class package: " + e.getMessage()); + } catch (NoSuchMethodException e) { + throw new IOException("Cannot get DistCp constructor: " + e.getMessage()); + } catch (Exception e) { + throw new IOException("Cannot execute DistCp process: " + e, e); + } + + return (0 == rc) ? true : false; + } + + @Override + public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException { + return new HadoopShims.NoopHdfsEncryptionShim(); + } } Index: shims/0.23/pom.xml =================================================================== --- shims/0.23/pom.xml (revision 1653459) +++ shims/0.23/pom.xml (working copy) @@ -145,4 +145,17 @@ test-jar + + + + hadoop-2 + + + org.apache.hadoop + hadoop-distcp + ${hadoop-23.version} + + + + Index: shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java =================================================================== --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (revision 1653459) +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (working copy) @@ -19,20 +19,27 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URI; import java.security.AccessControlException; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.Options; +import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.DefaultFileAccess; import org.apache.hadoop.fs.FSDataInputStream; @@ -53,7 +60,12 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; @@ -75,6 +87,7 @@ import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.test.MiniTezCluster; @@ -89,17 +102,29 @@ public class Hadoop23Shims extends HadoopShimsSecure { HadoopShims.MiniDFSShim cluster = null; - final boolean zeroCopy; + final boolean storagePolicy; public Hadoop23Shims() { boolean zcr = false; + boolean storage = false; try { Class.forName("org.apache.hadoop.fs.CacheFlag", false, ShimLoader.class.getClassLoader()); zcr = true; } catch (ClassNotFoundException ce) { } + + if (zcr) { + // in-memory HDFS is only available after zcr + try { + Class.forName("org.apache.hadoop.hdfs.protocol.BlockStoragePolicy", + false, ShimLoader.class.getClassLoader()); + storage = true; + } catch (ClassNotFoundException ce) { + } + } + this.storagePolicy = storage; this.zeroCopy = zcr; } @@ -373,7 +398,14 @@ int numDataNodes, boolean format, String[] racks) throws IOException { - cluster = new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks)); + MiniDFSCluster miniDFSCluster = new MiniDFSCluster(conf, numDataNodes, format, racks); + + // Need to set the client's KeyProvider to the NN's for JKS, + // else the updates do not get flushed properly + miniDFSCluster.getFileSystem().getClient().setKeyProvider( + miniDFSCluster.getNameNode().getNamesystem().getProvider()); + + cluster = new MiniDFSShim(miniDFSCluster); return cluster; } @@ -935,4 +967,224 @@ return kerberosName.getShortName(); } } + + + public static class StoragePolicyShim implements HadoopShims.StoragePolicyShim { + + private final DistributedFileSystem dfs; + + public StoragePolicyShim(DistributedFileSystem fs) { + this.dfs = fs; + } + + @Override + public void setStoragePolicy(Path path, StoragePolicyValue policy) + throws IOException { + switch (policy) { + case MEMORY: { + dfs.setStoragePolicy(path, HdfsConstants.MEMORY_STORAGE_POLICY_NAME); + break; + } + case SSD: { + dfs.setStoragePolicy(path, HdfsConstants.ALLSSD_STORAGE_POLICY_NAME); + break; + } + case DEFAULT: { + /* do nothing */ + break; + } + default: + throw new IllegalArgumentException("Unknown storage policy " + policy); + } + } + } + + + @Override + public HadoopShims.StoragePolicyShim getStoragePolicyShim(FileSystem fs) { + if (!storagePolicy) { + return null; + } + try { + return new StoragePolicyShim((DistributedFileSystem) fs); + } catch (ClassCastException ce) { + return null; + } + } + + @Override + public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException { + int rc; + + // Creates the command-line parameters for distcp + String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()}; + + try { + Class clazzDistCp = Class.forName("org.apache.hadoop.tools.DistCp"); + Constructor c = clazzDistCp.getConstructor(); + c.setAccessible(true); + Tool distcp = (Tool)c.newInstance(); + distcp.setConf(conf); + rc = distcp.run(params); + } catch (ClassNotFoundException e) { + throw new IOException("Cannot find DistCp class package: " + e.getMessage()); + } catch (NoSuchMethodException e) { + throw new IOException("Cannot get DistCp constructor: " + e.getMessage()); + } catch (Exception e) { + throw new IOException("Cannot execute DistCp process: " + e, e); + } + + return (0 == rc); + } + + public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { + private final String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; + + /** + * Gets information about HDFS encryption zones + */ + private HdfsAdmin hdfsAdmin = null; + + /** + * Used to compare encryption key strengths. + */ + private KeyProvider keyProvider = null; + + private Configuration conf; + + public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { + DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf); + + this.conf = conf; + this.keyProvider = dfs.getClient().getKeyProvider(); + this.hdfsAdmin = new HdfsAdmin(uri, conf); + } + + @Override + public boolean isPathEncrypted(Path path) throws IOException { + Path fullPath; + if (path.isAbsolute()) { + fullPath = path; + } else { + fullPath = path.getFileSystem(conf).makeQualified(path); + } + return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); + } + + @Override + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); + zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return true; + } else if (zone1 == null || zone2 == null) { + return false; + } + + return zone1.equals(zone2); + } + + @Override + public int comparePathKeyStrength(Path path1, Path path2) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); + zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return 0; + } else if (zone1 == null) { + return -1; + } else if (zone2 == null) { + return 1; + } + + return compareKeyStrength(zone1.getKeyName(), zone2.getKeyName()); + } + + @Override + public void createEncryptionZone(Path path, String keyName) throws IOException { + hdfsAdmin.createEncryptionZone(path, keyName); + } + + @Override + public void createKey(String keyName, int bitLength) + throws IOException, NoSuchAlgorithmException { + + checkKeyProvider(); + + if (keyProvider.getMetadata(keyName) == null) { + final KeyProvider.Options options = new Options(this.conf); + options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); + options.setBitLength(bitLength); + keyProvider.createKey(keyName, options); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' already exists"); + } + } + + @Override + public void deleteKey(String keyName) throws IOException { + checkKeyProvider(); + + if (keyProvider.getMetadata(keyName) != null) { + keyProvider.deleteKey(keyName); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' does not exist."); + } + } + + @Override + public List getKeys() throws IOException { + checkKeyProvider(); + return keyProvider.getKeys(); + } + + private void checkKeyProvider() throws IOException { + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + } + + /** + * Compares two encryption key strengths. + * + * @param keyname1 Keyname to compare + * @param keyname2 Keyname to compare + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get key metadata + */ + private int compareKeyStrength(String keyname1, String keyname2) throws IOException { + KeyProvider.Metadata meta1, meta2; + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + + meta1 = keyProvider.getMetadata(keyname1); + meta2 = keyProvider.getMetadata(keyname2); + + if (meta1.getBitLength() < meta2.getBitLength()) { + return -1; + } else if (meta1.getBitLength() == meta2.getBitLength()) { + return 0; + } else { + return 1; + } + } + } + + @Override + public HadoopShims.HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException { + URI uri = fs.getUri(); + if ("hdfs".equals(uri.getScheme())) { + return new HdfsEncryptionShim(uri, conf); + } + return new HadoopShims.NoopHdfsEncryptionShim(); + } } Index: shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java =================================================================== --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (revision 1653459) +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (working copy) @@ -23,11 +23,19 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedExceptionAction; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import javax.security.auth.login.LoginException; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; @@ -38,6 +46,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyValue; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; @@ -388,8 +397,35 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri); public Map getHadoopConfNames(); + + /** + * Create a shim for DFS storage policy. + */ + + public enum StoragePolicyValue { + MEMORY, /* 1-replica memory */ + SSD, /* 3-replica ssd */ + DEFAULT /* system defaults (usually 3-replica disk) */; + public static StoragePolicyValue lookup(String name) { + if (name == null) { + return DEFAULT; + } + return StoragePolicyValue.valueOf(name.toUpperCase().trim()); + } + }; + + public interface StoragePolicyShim { + void setStoragePolicy(Path path, StoragePolicyValue policy) throws IOException; + } + /** + * obtain a storage policy shim associated with the filesystem. + * Returns null when the filesystem has no storage policies. + */ + public StoragePolicyShim getStoragePolicyShim(FileSystem fs); + + /** * a hadoop.io ByteBufferPool shim. */ public interface ByteBufferPoolShim { @@ -538,4 +574,131 @@ public String getShortName() throws IOException; } + /** + * Copies a source dir/file to a destination by orchestrating the copy between hdfs nodes. + * This distributed process is meant to copy huge files that could take some time if a single + * copy is done. + * + * @param src Path to the source file or directory to copy + * @param dst Path to the destination file or directory + * @param conf The hadoop configuration object + * @return True if it is successfull; False otherwise. + */ + public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException; + + /** + * This interface encapsulates methods used to get encryption information from + * HDFS paths. + */ + public interface HdfsEncryptionShim { + /** + * Checks if a given HDFS path is encrypted. + * + * @param path Path to HDFS file system + * @return True if it is encrypted; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public boolean isPathEncrypted(Path path) throws IOException; + + /** + * Checks if two HDFS paths are on the same encrypted or unencrypted zone. + * + * @param path1 Path to HDFS file system + * @param path2 Path to HDFS file system + * @return True if both paths are in the same zone; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException; + + /** + * Compares two encrypted path strengths. + * + * @param path1 HDFS path to compare. + * @param path2 HDFS path to compare. + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get encryption/key metadata + */ + public int comparePathKeyStrength(Path path1, Path path2) throws IOException; + + /** + * create encryption zone by path and keyname + * @param path HDFS path to create encryption zone + * @param keyName keyname + * @throws IOException + */ + @VisibleForTesting + public void createEncryptionZone(Path path, String keyName) throws IOException; + + /** + * Creates an encryption key. + * + * @param keyName Name of the key + * @param bitLength Key encryption length in bits (128 or 256). + * @throws IOException If an error occurs while creating the encryption key + * @throws NoSuchAlgorithmException If cipher algorithm is invalid. + */ + @VisibleForTesting + public void createKey(String keyName, int bitLength) + throws IOException, NoSuchAlgorithmException; + + @VisibleForTesting + public void deleteKey(String keyName) throws IOException; + + @VisibleForTesting + public List getKeys() throws IOException; + } + + /** + * This is a dummy class used when the hadoop version does not support hdfs encryption. + */ + public static class NoopHdfsEncryptionShim implements HdfsEncryptionShim { + @Override + public boolean isPathEncrypted(Path path) throws IOException { + /* not supported */ + return false; + } + + @Override + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { + /* not supported */ + return true; + } + + @Override + public int comparePathKeyStrength(Path path1, Path path2) throws IOException { + /* not supported */ + return 0; + } + + @Override + public void createEncryptionZone(Path path, String keyName) { + /* not supported */ + } + + @Override + public void createKey(String keyName, int bitLength) { + /* not supported */ + } + + @Override + public void deleteKey(String keyName) throws IOException { + /* not supported */ + } + + @Override + public List getKeys() throws IOException{ + /* not supported */ + return null; + } + } + + /** + * Returns a new instance of the HdfsEncryption shim. + * + * @param fs A FileSystem object to HDFS + * @param conf A Configuration object + * @return A new instance of the HdfsEncryption shim. + * @throws IOException If an error occurred while creating the instance. + */ + public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException; } Index: shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java =================================================================== --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (revision 1653459) +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (working copy) @@ -24,10 +24,8 @@ import java.net.URI; import java.security.AccessControlException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.commons.lang.ArrayUtils; @@ -53,8 +51,6 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.Progressable; -import com.google.common.primitives.Longs; - /** * Base implemention for shims against secure Hadoop 0.20.3/0.23. */ @@ -325,16 +321,9 @@ ArrayList inputSplitShims = new ArrayList(); for (int pos = 0; pos < splits.length; pos++) { CombineFileSplit split = (CombineFileSplit) splits[pos]; - Set dirIndices = getDirIndices(split.getPaths(), job); - if (dirIndices.size() != split.getPaths().length) { - List prunedPaths = prune(dirIndices, Arrays.asList(split.getPaths())); - List prunedStartOffsets = prune(dirIndices, Arrays.asList( - ArrayUtils.toObject(split.getStartOffsets()))); - List prunedLengths = prune(dirIndices, Arrays.asList( - ArrayUtils.toObject(split.getLengths()))); - inputSplitShims.add(new InputSplitShim(job, prunedPaths.toArray(new Path[prunedPaths.size()]), - Longs.toArray(prunedStartOffsets), - Longs.toArray(prunedLengths), split.getLocations())); + if (split.getPaths().length > 0) { + inputSplitShims.add(new InputSplitShim(job, split.getPaths(), + split.getStartOffsets(), split.getLengths(), split.getLocations())); } } return inputSplitShims.toArray(new InputSplitShim[inputSplitShims.size()]); @@ -354,6 +343,27 @@ return new CombineFileRecordReader(job, cfSplit, reporter, rrClass); } + @Override + protected FileStatus[] listStatus(JobConf job) throws IOException { + FileStatus[] result = super.listStatus(job); + boolean foundDir = false; + for (FileStatus stat: result) { + if (stat.isDir()) { + foundDir = true; + break; + } + } + if (!foundDir) { + return result; + } + ArrayList files = new ArrayList(); + for (FileStatus stat: result) { + if (!stat.isDir()) { + files.add(stat); + } + } + return files.toArray(new FileStatus[files.size()]); + } } @Override @@ -400,33 +410,6 @@ LOG.debug("Return value is :" + retval); } - /** - * CombineFileInputFormat sometimes returns directories as splits, need to prune them. - */ - private static Set getDirIndices(Path[] paths, JobConf conf) throws IOException { - Set result = new HashSet(); - for (int i = 0; i < paths.length; i++) { - FileSystem fs = paths[i].getFileSystem(conf); - if (!fs.isFile(paths[i])) { - result.add(i); - } - } - return result; - } - - private static List prune(Set indicesToPrune, List elms) { - List result = new ArrayList(); - int i = 0; - for (K elm : elms) { - if (indicesToPrune.contains(i)) { - continue; - } - result.add(elm); - i++; - } - return result; - } - private static String[] dedup(String[] locations) throws IOException { Set dedup = new HashSet(); Collections.addAll(dedup, locations); Index: spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java =================================================================== --- spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java (revision 1653459) +++ spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java (working copy) @@ -50,7 +50,7 @@ public class TestSparkClient { // Timeouts are bad... mmmkay. - private static final long TIMEOUT = 10; + private static final long TIMEOUT = 20; private static final HiveConf HIVECONF = new HiveConf(); private Map createConf(boolean local) {