diff --git a/data/files/load_data_job/bucketing.txt b/data/files/load_data_job/bucketing.txt new file mode 100644 index 0000000000..83344151c2 --- /dev/null +++ b/data/files/load_data_job/bucketing.txt @@ -0,0 +1,118 @@ +165val_165 +484val_484 +150val_150 +224val_224 +66val_66 +213val_213 +374val_374 +495val_495 +37val_37 +327val_327 +15val_15 +338val_338 +459val_459 +466val_466 +396val_396 +309val_309 +367val_367 +0val_0 +455val_455 +316val_316 +345val_345 +129val_129 +378val_378 +4val_4 +356val_356 +169val_169 +125val_125 +437val_437 +286val_286 +187val_187 +176val_176 +459val_459 +51val_51 +103val_103 +239val_239 +213val_213 +176val_176 +275val_275 +260val_260 +404val_404 +217val_217 +84val_84 +466val_466 +8val_8 +411val_411 +172val_172 +129val_129 +158val_158 +0val_0 +26val_26 +165val_165 +327val_327 +51val_51 +404val_404 +95val_95 +282val_282 +187val_187 +316val_316 +169val_169 +77val_77 +0val_0 +118val_118 +282val_282 +419val_419 +15val_15 +118val_118 +19val_19 +224val_224 +309val_309 +389val_389 +327val_327 +242val_242 +392val_392 +242val_242 +396val_396 +95val_95 +11val_11 +143val_143 +228val_228 +33val_33 +103val_103 +367val_367 +239val_239 +480val_480 +202val_202 +316val_316 +235val_235 +80val_80 +44val_44 +466val_466 +257val_257 +190val_190 +114val_114 +396val_396 +217val_217 +125val_125 +187val_187 +480val_480 +491val_491 +305val_305 +444val_444 +169val_169 +323val_323 +480val_480 +136val_136 +172val_172 +462val_462 +26val_26 +462val_462 +341val_341 +183val_183 +84val_84 +37val_37 +448val_448 +194val_194 +477val_477 +169val_169 +400val_400 diff --git a/data/files/load_data_job/load_data_1_partition.txt b/data/files/load_data_job/load_data_1_partition.txt new file mode 100644 index 0000000000..9f34604c4c --- /dev/null +++ b/data/files/load_data_job/load_data_1_partition.txt @@ -0,0 +1,118 @@ +165val_1652008-04-08 +484val_4842008-04-08 +150val_1502008-04-08 +224val_2242008-04-08 +66val_662008-04-08 +213val_2132008-04-08 +374val_3742008-04-08 +495val_4952008-04-08 +37val_372008-04-08 +327val_3272008-04-08 +15val_152008-04-08 +338val_3382008-04-08 +459val_4592008-04-08 +466val_4662008-04-08 +396val_3962008-04-08 +309val_3092008-04-08 +367val_3672008-04-08 +0val_02008-04-08 +455val_4552008-04-08 +316val_3162008-04-08 +345val_3452008-04-08 +129val_1292008-04-08 +378val_3782008-04-08 +4val_42008-04-08 +356val_3562008-04-08 +169val_1692008-04-08 +125val_1252008-04-08 +437val_4372008-04-08 +286val_2862008-04-08 +187val_1872008-04-08 +176val_1762008-04-08 +459val_4592008-04-08 +51val_512008-04-08 +103val_1032008-04-08 +239val_2392008-04-08 +213val_2132008-04-08 +176val_1762008-04-08 +275val_2752008-04-08 +260val_2602008-04-08 +404val_4042008-04-08 +217val_2172008-04-08 +84val_842008-04-08 +466val_4662008-04-08 +8val_82008-04-08 +411val_4112008-04-08 +172val_1722008-04-08 +129val_1292008-04-08 +158val_1582008-04-08 +0val_02008-04-08 +26val_262008-04-08 +165val_1652008-04-08 +327val_3272008-04-08 +51val_512008-04-08 +404val_4042008-04-08 +95val_952008-04-08 +282val_2822008-04-08 +187val_1872008-04-08 +316val_3162008-04-08 +169val_1692008-04-08 +77val_772008-04-08 +0val_02008-04-08 +118val_1182008-04-08 +282val_2822008-04-08 +419val_4192008-04-08 +15val_152008-04-08 +118val_1182008-04-08 +19val_192008-04-08 +224val_2242008-04-08 +309val_3092008-04-08 +389val_3892008-04-08 +327val_3272008-04-08 +242val_2422008-04-08 +392val_3922008-04-08 +242val_2422008-04-08 +396val_3962008-04-08 +95val_952008-04-08 +11val_112008-04-08 +143val_1432008-04-08 +228val_2282008-04-08 +33val_332008-04-08 +103val_1032008-04-08 +367val_3672008-04-08 +239val_2392008-04-08 +480val_4802008-04-08 +202val_2022008-04-08 +316val_3162008-04-08 +235val_2352008-04-08 +80val_802008-04-08 +44val_442008-04-08 +466val_4662008-04-08 +257val_2572008-04-08 +190val_1902008-04-08 +114val_1142008-04-08 +396val_3962008-04-08 +217val_2172008-04-08 +125val_1252008-04-08 +187val_1872008-04-08 +480val_4802008-04-08 +491val_4912008-04-08 +305val_3052008-04-08 +444val_4442008-04-08 +169val_1692008-04-08 +323val_3232008-04-08 +480val_4802008-04-08 +136val_1362008-04-08 +172val_1722008-04-08 +462val_4622008-04-08 +26val_262008-04-08 +462val_4622008-04-08 +341val_3412008-04-08 +183val_1832008-04-08 +84val_842008-04-08 +37val_372008-04-08 +448val_4482008-04-08 +194val_1942008-04-08 +477val_4772008-04-08 +169val_1692008-04-08 +400val_4002008-04-08 diff --git a/data/files/load_data_job/partitions/load_data_1_partition.txt b/data/files/load_data_job/partitions/load_data_1_partition.txt new file mode 100644 index 0000000000..50ea6d5637 --- /dev/null +++ b/data/files/load_data_job/partitions/load_data_1_partition.txt @@ -0,0 +1,118 @@ +165val_1652008-04-081 +484val_4842008-04-080 +150val_1502008-04-081 +224val_2242008-04-080 +66val_662008-04-081 +213val_2132008-04-080 +374val_3742008-04-081 +495val_4952008-04-080 +37val_372008-04-081 +327val_3272008-04-080 +15val_152008-04-081 +338val_3382008-04-080 +459val_4592008-04-081 +466val_4662008-04-080 +396val_3962008-04-081 +309val_3092008-04-080 +367val_3672008-04-081 +0val_02008-04-080 +455val_4552008-04-081 +316val_3162008-04-080 +345val_3452008-04-081 +129val_1292008-04-080 +378val_3782008-04-081 +4val_42008-04-080 +356val_3562008-04-081 +169val_1692008-04-080 +125val_1252008-04-081 +437val_4372008-04-080 +286val_2862008-04-081 +187val_1872008-04-080 +176val_1762008-04-081 +459val_4592008-04-080 +51val_512008-04-081 +103val_1032008-04-080 +239val_2392008-04-081 +213val_2132008-04-080 +176val_1762008-04-081 +275val_2752008-04-080 +260val_2602008-04-081 +404val_4042008-04-080 +217val_2172008-04-081 +84val_842008-04-080 +466val_4662008-04-081 +8val_82008-04-080 +411val_4112008-04-081 +172val_1722008-04-080 +129val_1292008-04-081 +158val_1582008-04-080 +0val_02008-04-081 +26val_262008-04-080 +165val_1652008-04-081 +327val_3272008-04-080 +51val_512008-04-081 +404val_4042008-04-080 +95val_952008-04-081 +282val_2822008-04-080 +187val_1872008-04-081 +316val_3162008-04-080 +169val_1692008-04-081 +77val_772008-04-080 +0val_02008-04-081 +118val_1182008-04-080 +282val_2822008-04-081 +419val_4192008-04-080 +15val_152008-04-081 +118val_1182008-04-080 +19val_192008-04-081 +224val_2242008-04-080 +309val_3092008-04-081 +389val_3892008-04-080 +327val_3272008-04-081 +242val_2422008-04-080 +392val_3922008-04-081 +242val_2422008-04-080 +396val_3962008-04-081 +95val_952008-04-080 +11val_112008-04-081 +143val_1432008-04-080 +228val_2282008-04-081 +33val_332008-04-080 +103val_1032008-04-081 +367val_3672008-04-080 +239val_2392008-04-081 +480val_4802008-04-080 +202val_2022008-04-081 +316val_3162008-04-080 +235val_2352008-04-081 +80val_802008-04-080 +44val_442008-04-081 +466val_4662008-04-080 +257val_2572008-04-081 +190val_1902008-04-080 +114val_1142008-04-081 +396val_3962008-04-080 +217val_2172008-04-081 +125val_1252008-04-080 +187val_1872008-04-081 +480val_4802008-04-080 +491val_4912008-04-081 +305val_3052008-04-080 +444val_4442008-04-081 +169val_1692008-04-080 +323val_3232008-04-081 +480val_4802008-04-080 +136val_1362008-04-081 +172val_1722008-04-080 +462val_4622008-04-081 +26val_262008-04-080 +462val_4622008-04-081 +341val_3412008-04-080 +183val_1832008-04-081 +84val_842008-04-080 +37val_372008-04-081 +448val_4482008-04-080 +194val_1942008-04-081 +477val_4772008-04-080 +169val_1692008-04-081 +400val_4002008-04-080 diff --git a/data/files/load_data_job/partitions/load_data_2_partitions.txt b/data/files/load_data_job/partitions/load_data_2_partitions.txt new file mode 100644 index 0000000000..50ea6d5637 --- /dev/null +++ b/data/files/load_data_job/partitions/load_data_2_partitions.txt @@ -0,0 +1,118 @@ +165val_1652008-04-081 +484val_4842008-04-080 +150val_1502008-04-081 +224val_2242008-04-080 +66val_662008-04-081 +213val_2132008-04-080 +374val_3742008-04-081 +495val_4952008-04-080 +37val_372008-04-081 +327val_3272008-04-080 +15val_152008-04-081 +338val_3382008-04-080 +459val_4592008-04-081 +466val_4662008-04-080 +396val_3962008-04-081 +309val_3092008-04-080 +367val_3672008-04-081 +0val_02008-04-080 +455val_4552008-04-081 +316val_3162008-04-080 +345val_3452008-04-081 +129val_1292008-04-080 +378val_3782008-04-081 +4val_42008-04-080 +356val_3562008-04-081 +169val_1692008-04-080 +125val_1252008-04-081 +437val_4372008-04-080 +286val_2862008-04-081 +187val_1872008-04-080 +176val_1762008-04-081 +459val_4592008-04-080 +51val_512008-04-081 +103val_1032008-04-080 +239val_2392008-04-081 +213val_2132008-04-080 +176val_1762008-04-081 +275val_2752008-04-080 +260val_2602008-04-081 +404val_4042008-04-080 +217val_2172008-04-081 +84val_842008-04-080 +466val_4662008-04-081 +8val_82008-04-080 +411val_4112008-04-081 +172val_1722008-04-080 +129val_1292008-04-081 +158val_1582008-04-080 +0val_02008-04-081 +26val_262008-04-080 +165val_1652008-04-081 +327val_3272008-04-080 +51val_512008-04-081 +404val_4042008-04-080 +95val_952008-04-081 +282val_2822008-04-080 +187val_1872008-04-081 +316val_3162008-04-080 +169val_1692008-04-081 +77val_772008-04-080 +0val_02008-04-081 +118val_1182008-04-080 +282val_2822008-04-081 +419val_4192008-04-080 +15val_152008-04-081 +118val_1182008-04-080 +19val_192008-04-081 +224val_2242008-04-080 +309val_3092008-04-081 +389val_3892008-04-080 +327val_3272008-04-081 +242val_2422008-04-080 +392val_3922008-04-081 +242val_2422008-04-080 +396val_3962008-04-081 +95val_952008-04-080 +11val_112008-04-081 +143val_1432008-04-080 +228val_2282008-04-081 +33val_332008-04-080 +103val_1032008-04-081 +367val_3672008-04-080 +239val_2392008-04-081 +480val_4802008-04-080 +202val_2022008-04-081 +316val_3162008-04-080 +235val_2352008-04-081 +80val_802008-04-080 +44val_442008-04-081 +466val_4662008-04-080 +257val_2572008-04-081 +190val_1902008-04-080 +114val_1142008-04-081 +396val_3962008-04-080 +217val_2172008-04-081 +125val_1252008-04-080 +187val_1872008-04-081 +480val_4802008-04-080 +491val_4912008-04-081 +305val_3052008-04-080 +444val_4442008-04-081 +169val_1692008-04-080 +323val_3232008-04-081 +480val_4802008-04-080 +136val_1362008-04-081 +172val_1722008-04-080 +462val_4622008-04-081 +26val_262008-04-080 +462val_4622008-04-081 +341val_3412008-04-080 +183val_1832008-04-081 +84val_842008-04-080 +37val_372008-04-081 +448val_4482008-04-080 +194val_1942008-04-081 +477val_4772008-04-080 +169val_1692008-04-081 +400val_4002008-04-080 diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 1a346593fd..4bce29804d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -546,6 +546,7 @@ minillaplocal.query.files=\ llap_smb.q,\ llap_vector_nohybridgrace.q,\ load_data_acid_rename.q,\ + load_data_using_job.q,\ load_dyn_part5.q,\ lvj_mapjoin.q,\ materialized_view_create.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 0fedf0e76e..1921ea7ca8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -162,6 +162,9 @@ private StatsSource statsSource; private int executionIndex; + // Load data rewrite + private Table tempTableForLoad; + public void setOperation(Operation operation) { this.operation = operation; } @@ -516,7 +519,6 @@ public Path getMRScratchDir() { * - If path is on HDFS, then create a staging directory inside the path * * @param path Path used to verify the Filesystem to use for temporary directory - * @param isFinalJob true if the required {@link Path} will be used for the final job (e.g. the final FSOP) * * @return A path to the new temporary directory */ @@ -1071,4 +1073,12 @@ public void setExecutionIndex(int executionIndex) { public void setConf(HiveConf conf) { this.conf = conf; } + + public Table getTempTableForLoad() { + return tempTableForLoad; + } + + public void setTempTableForLoad(Table tempTableForLoad) { + this.tempTableForLoad = tempTableForLoad; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 7d33fa3892..84d778c176 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -462,6 +462,8 @@ true), ACID_OP_ON_INSERTONLYTRAN_TABLE(10414, "Attempt to do update or delete on table {0} that is " + "insert-only transactional", true), + LOAD_DATA_LAUNCH_JOB_IO_ERROR(10415, "Encountered I/O error while parsing rewritten load data into insert query"), + LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"), //========================== 20000 range starts here ========================// diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index a51fdd322f..b792851c18 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -519,7 +519,7 @@ public FieldSchema getPartColByName(String colName) { } public boolean isPartitionKey(String colName) { - return getPartColByName(colName) == null ? false : true; + return getPartColByName(colName) != null; } // TODO merge this with getBucketCols function @@ -1009,8 +1009,8 @@ public static boolean shouldStoreFieldsInMetastore( public static void validateColumns(List columns, List partCols) throws HiveException { Set colNames = new HashSet<>(); - for (FieldSchema partCol: columns) { - String colName = normalize(partCol.getName()); + for (FieldSchema col: columns) { + String colName = normalize(col.getName()); if (colNames.contains(colName)) { throw new HiveException("Duplicate column name " + colName + " in the table definition."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index c07991d434..8d71711359 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -27,6 +27,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; import org.antlr.runtime.tree.Tree; import org.apache.commons.httpclient.util.URIUtil; @@ -38,6 +39,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.Task; @@ -56,16 +58,26 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.BasicStatsWork; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.InputFormat; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * LoadSemanticAnalyzer. * */ -public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer { +public class LoadSemanticAnalyzer extends SemanticAnalyzer { + private static final Logger LOG = LoggerFactory.getLogger(LoadSemanticAnalyzer.class); + private boolean useSuper = false; + private boolean queryReWritten = false; + + private final String tempTblNameSuffix = "__TEMP_TABLE_FOR_LOAD_DATA__"; + + // AST specific data + private Tree fromTree, tableTree; + private boolean isLocal = false, isOverWrite = false; public LoadSemanticAnalyzer(QueryState queryState) throws SemanticException { super(queryState); @@ -77,7 +89,7 @@ public LoadSemanticAnalyzer(QueryState queryState) throws SemanticException { @Override public boolean accept(Path p) { String name = p.getName(); - return name.equals(EximUtil.METADATA_NAME) ? true : !name.startsWith("_") && !name.startsWith("."); + return name.equals(EximUtil.METADATA_NAME) || (!name.startsWith("_") && !name.startsWith(".")); } }); if ((srcs != null) && srcs.length == 1) { @@ -137,15 +149,14 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti return new URI(fromScheme, fromAuthority, path, null, null); } - private List applyConstraintsAndGetFiles(URI fromURI, Tree ast, - boolean isLocal, Table table) throws SemanticException { + private List applyConstraintsAndGetFiles(URI fromURI, Table table) throws SemanticException { FileStatus[] srcs = null; // local mode implies that scheme should be "file" // we can change this going forward if (isLocal && !fromURI.getScheme().equals("file")) { - throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(ast, + throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(fromTree, "Source file system should be \"file\" if \"local\" is specified")); } @@ -153,13 +164,13 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti FileSystem fileSystem = FileSystem.get(fromURI, conf); srcs = matchFilesOrDir(fileSystem, new Path(fromURI)); if (srcs == null || srcs.length == 0) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, "No files matching path " + fromURI)); } for (FileStatus oneSrc : srcs) { if (oneSrc.isDir()) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, "source contains directory: " + oneSrc.getPath().toString())); } } @@ -184,44 +195,17 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti int bucketId = Utilities.getBucketIdFromFile(bucketIdStr); LOG.debug("bucket ID for file " + oneSrc.getPath() + " = " + bucketId + " for table " + table.getFullyQualifiedName()); - if (bucketId == -1) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( - "The file name is invalid : " - + oneSrc.getPath().toString() + " for table " - + table.getFullyQualifiedName())); - } - if (bucketId >= numBuckets) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( - "The file name corresponds to invalid bucketId : " - + oneSrc.getPath().toString()) - + ". Maximum number of buckets can be " + numBuckets - + " for table " + table.getFullyQualifiedName()); - } - if (bucketArray[bucketId]) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( - "Multiple files for same bucket : " + bucketId - + ". Only 1 file per bucket allowed in single load command. To load " + - "multiple files for same bucket, use multiple statements for table " - + table.getFullyQualifiedName())); + if (bucketId == -1 || bucketId >= numBuckets || bucketArray[bucketId]) { + reparseAndSuperAnalyze(table, fromURI); + return null; } bucketArray[bucketId] = true; } } - else { - /** - * for loading into un-bucketed acid table, files can be named arbitrarily but they will - * be renamed during load. - * {@link Hive#mvFile(HiveConf, FileSystem, Path, FileSystem, Path, boolean, boolean, - * boolean, int)} - * and - * {@link Hive#copyFiles(HiveConf, FileSystem, FileStatus[], FileSystem, Path, boolean, - * boolean, List, boolean)} - */ - } } catch (IOException e) { // Has to use full name to make sure it does not conflict with // org.apache.commons.lang.StringUtils - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast), e); + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree), e); } return Lists.newArrayList(srcs); @@ -249,12 +233,31 @@ private static void validateAcidFiles(Table table, FileStatus[] srcs, FileSystem } } + @Override + public void init(boolean clearPartsCache) { + // Unless in super mode, skip the whole thing + if (useSuper) { + super.init(clearPartsCache); + + Table tempTable = ctx.getTempTableForLoad(); + if (tempTable != null) { + tabNameToTabObject.put(tempTable.getTableName().toLowerCase(), tempTable); + } + } + } + @Override public void analyzeInternal(ASTNode ast) throws SemanticException { - boolean isLocal = false; - boolean isOverWrite = false; - Tree fromTree = ast.getChild(0); - Tree tableTree = ast.getChild(1); + if (useSuper) { + super.analyzeInternal(ast); + } else { + analyzeLoad(ast); + } + } + + private void analyzeLoad(ASTNode ast) throws SemanticException { + fromTree = ast.getChild(0); + tableTree = ast.getChild(1); if (ast.getChildCount() == 4) { isLocal = true; @@ -298,20 +301,24 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { List parts = ts.tableHandle.getPartitionKeys(); if ((parts != null && parts.size() > 0) && (ts.partSpec == null || ts.partSpec.size() == 0)) { - throw new SemanticException(ErrorMsg.NEED_PARTITION_ERROR.getMsg()); + // launch a tez job + reparseAndSuperAnalyze(ts.tableHandle, fromURI); + return; } List bucketCols = ts.tableHandle.getBucketCols(); if (bucketCols != null && !bucketCols.isEmpty()) { String error = StrictChecks.checkBucketing(conf); if (error != null) { - throw new SemanticException("Please load into an intermediate table" - + " and use 'insert... select' to allow Hive to enforce bucketing. " + error); + // launch a tez job + reparseAndSuperAnalyze(ts.tableHandle, fromURI); + return; } } // make sure the arguments make sense - List files = applyConstraintsAndGetFiles(fromURI, fromTree, isLocal, ts.tableHandle); + List files = applyConstraintsAndGetFiles(fromURI, ts.tableHandle); + if (queryReWritten) return; // for managed tables, make sure the file formats match if (TableType.MANAGED_TABLE.equals(ts.tableHandle.getTableType()) @@ -430,4 +437,70 @@ private void ensureFileFormatsMatch(TableSpec ts, List fileStatuses, " Error: " + e.getMessage()); } } + + // Rewrite the load to launch an insert job. + private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticException { + LOG.info("Load data triggered a Tez job instead of usual file operation"); + // Step 1 : Create a temp table object + // Create a Table object + Table tempTableObj = new Table(new org.apache.hadoop.hive.metastore.api.Table(table.getTTable())); + // Construct a temp table name + String tempTblName = table.getTableName() + tempTblNameSuffix; + tempTableObj.setTableName(tempTblName); + + // Move all the partition columns at the end of table columns + tempTableObj.setFields(table.getAllCols()); + // wipe out partition columns + tempTableObj.setPartCols(new ArrayList<>()); + + // Set data location + tempTableObj.setDataLocation(new Path(fromURI)); + + // Step 2 : create the Insert query + StringBuilder rewrittenQueryStr = new StringBuilder(); + + rewrittenQueryStr.append("insert into table "); + rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0)))); + addPartitionColsToInsert(table.getPartCols(), rewrittenQueryStr); + rewrittenQueryStr.append(" select * from "); + rewrittenQueryStr.append(tempTblName); + + // Step 3 : parse the query + // Set dynamic partitioning to nonstrict so that queries do not need any partition + // references. + HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + // Parse the rewritten query string + Context rewrittenCtx; + try { + rewrittenCtx = new Context(conf); + // We keep track of all the contexts that are created by this query + // so we can clear them when we finish execution + ctx.addRewrittenStatementContext(rewrittenCtx); + } catch (IOException e) { + throw new SemanticException(ErrorMsg.LOAD_DATA_LAUNCH_JOB_IO_ERROR.getMsg()); + } + rewrittenCtx.setExplainConfig(ctx.getExplainConfig()); + rewrittenCtx.setExplainPlan(ctx.isExplainPlan()); + rewrittenCtx.setIsUpdateDeleteMerge(true); + rewrittenCtx.setCmd(rewrittenQueryStr.toString()); + rewrittenCtx.setTempTableForLoad(tempTableObj); + + ASTNode rewrittenTree; + try { + LOG.info("Going to reparse <" + ctx.getCmd() + "> as \n<" + rewrittenQueryStr.toString() + ">"); + rewrittenTree = ParseUtils.parse(rewrittenQueryStr.toString(), rewrittenCtx); + } catch (ParseException e) { + throw new SemanticException(ErrorMsg.LOAD_DATA_LAUNCH_JOB_PARSE_ERROR.getMsg(), e); + } + + // Step 4 : Reanalyze + try { + useSuper = true; + super.analyze(rewrittenTree, rewrittenCtx); + } finally { + useSuper = false; + } + + queryReWritten = true; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1dccf969ff..6993c699e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -420,7 +420,6 @@ public SemanticAnalyzer(QueryState queryState) throws SemanticException { listMapJoinOpsNoReducer = new ArrayList>(); groupOpToInputTables = new HashMap>(); prunedPartitions = new HashMap(); - tabNameToTabObject = new HashMap(); unparseTranslator = new UnparseTranslator(conf); autogenColAliasPrfxLbl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL); @@ -1250,6 +1249,10 @@ private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) @Override public HashSet getAllOutputs() { + if (this instanceof LoadSemanticAnalyzer) { + // If this is LoadSemanticAnalyzer, make a shallow copy. + return super.getAllOutputs(); + } HashSet writeEntities = new HashSet(getOutputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { @@ -14767,4 +14770,46 @@ public boolean isValidQueryMaterialization() { AGGREGATE_REBUILD, NO_AGGREGATE_REBUILD } + + /** + * @return table name in db.table form with proper quoting/escaping to be used in a SQL statement + */ + protected String getFullTableNameForSQL(ASTNode n) throws SemanticException { + switch (n.getType()) { + case HiveParser.TOK_TABNAME: + String[] tableName = getQualifiedTableName(n); + return getDotName(new String[] { + HiveUtils.unparseIdentifier(tableName[0], this.conf), + HiveUtils.unparseIdentifier(tableName[1], this.conf) }); + case HiveParser.TOK_TABREF: + return getFullTableNameForSQL((ASTNode) n.getChild(0)); + default: + throw raiseWrongType("TOK_TABNAME", n); + } + } + + protected static IllegalArgumentException raiseWrongType(String expectedTokName, ASTNode n) { + return new IllegalArgumentException("Expected " + expectedTokName + "; got " + n.getType()); + } + + /** + * Append list of partition columns to Insert statement, i.e. the 1st set of partCol1,partCol2 + * INSERT INTO T PARTITION(partCol1,partCol2...) SELECT col1, ... partCol1,partCol2... + */ + protected void addPartitionColsToInsert(List partCols, StringBuilder rewrittenQueryStr) { + // If the table is partitioned we have to put the partition() clause in + if (partCols != null && partCols.size() > 0) { + rewrittenQueryStr.append(" partition ("); + boolean first = true; + for (FieldSchema fschema : partCols) { + if (first) + first = false; + else + rewrittenQueryStr.append(", "); + //would be nice if there was a way to determine if quotes are needed + rewrittenQueryStr.append(HiveUtils.unparseIdentifier(fschema.getName(), this.conf)); + } + rewrittenQueryStr.append(")"); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 2f3b07f4af..512f1ff3da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -350,26 +350,7 @@ private void analyzeDelete(ASTNode tree) throws SemanticException { currentOperation = Context.Operation.DELETE; reparseAndSuperAnalyze(tree); } - /** - * Append list of partition columns to Insert statement, i.e. the 1st set of partCol1,partCol2 - * INSERT INTO T PARTITION(partCol1,partCol2...) SELECT col1, ... partCol1,partCol2... - */ - private void addPartitionColsToInsert(List partCols, StringBuilder rewrittenQueryStr) { - // If the table is partitioned we have to put the partition() clause in - if (partCols != null && partCols.size() > 0) { - rewrittenQueryStr.append(" partition ("); - boolean first = true; - for (FieldSchema fschema : partCols) { - if (first) - first = false; - else - rewrittenQueryStr.append(", "); - //would be nice if there was a way to determine if quotes are needed - rewrittenQueryStr.append(HiveUtils.unparseIdentifier(fschema.getName(), this.conf)); - } - rewrittenQueryStr.append(")"); - } - } + /** * Append list of partition columns to Insert statement, i.e. the 2nd set of partCol1,partCol2 * INSERT INTO T PARTITION(partCol1,partCol2...) SELECT col1, ... partCol1,partCol2... @@ -1261,22 +1242,8 @@ private String getSimpleTableNameBase(ASTNode n) throws SemanticException { throw raiseWrongType("TOK_TABREF|TOK_TABNAME|TOK_SUBQUERY", n); } } - /** - * @return table name in db.table form with proper quoting/escaping to be used in a SQL statement - */ - private String getFullTableNameForSQL(ASTNode n) throws SemanticException { - switch (n.getType()) { - case HiveParser.TOK_TABNAME: - String[] tableName = getQualifiedTableName(n); - return getDotName(new String[] { - HiveUtils.unparseIdentifier(tableName[0], this.conf), - HiveUtils.unparseIdentifier(tableName[1], this.conf) }); - case HiveParser.TOK_TABREF: - return getFullTableNameForSQL((ASTNode) n.getChild(0)); - default: - throw raiseWrongType("TOK_TABNAME", n); - } - } private static final class ReparseResult { + + private static final class ReparseResult { private final ASTNode rewrittenTree; private final Context rewrittenCtx; ReparseResult(ASTNode n, Context c) { @@ -1284,9 +1251,7 @@ private String getFullTableNameForSQL(ASTNode n) throws SemanticException { rewrittenCtx = c; } } - private static IllegalArgumentException raiseWrongType(String expectedTokName, ASTNode n) { - return new IllegalArgumentException("Expected " + expectedTokName + "; got " + n.getType()); - } + private boolean isAliased(ASTNode n) { switch (n.getType()) { case HiveParser.TOK_TABREF: diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java index ec8c1507ec..11c59309fb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java @@ -352,6 +352,8 @@ public void loadDataPartitioned() throws Exception { * if loaded data is not bucketed properly. This test is to capture that this is still the default. * If the default is changed, Load Data should probably do more validation to ensure data is * properly distributed into files and files are named correctly. + * With the availability of new feature to rewrite such "load data" commands into insert-as-select, + * the test should let the load data pass. */ @Test public void testValidations() throws Exception { @@ -364,7 +366,8 @@ public void testValidations() throws Exception { //this creates an ORC data file with correct schema under table root runStatementOnDriver("insert into Tstage values(1,2),(3,4)"); CommandProcessorResponse cpr = runStatementOnDriverNegative("load data local inpath '" + getWarehouseDir() + "' into table T"); - Assert.assertTrue(cpr.getErrorMessage().contains("Load into bucketed tables are disabled")); + // This condition should not occur with the new support of rewriting load into IAS. + Assert.assertFalse(cpr.getErrorMessage().contains("Load into bucketed tables are disabled")); } private void checkExpected(List rs, String[][] expected, String msg) { super.checkExpected(rs, expected, msg, LOG, true); diff --git a/ql/src/test/queries/clientnegative/load_part_nospec.q b/ql/src/test/queries/clientnegative/load_part_nospec.q deleted file mode 100644 index 81517991b2..0000000000 --- a/ql/src/test/queries/clientnegative/load_part_nospec.q +++ /dev/null @@ -1,2 +0,0 @@ -create table hive_test_src ( col1 string ) partitioned by (pcol1 string) stored as textfile; -load data local inpath '../../data/files/test.dat' into table hive_test_src; diff --git a/ql/src/test/queries/clientnegative/nopart_load.q b/ql/src/test/queries/clientnegative/nopart_load.q deleted file mode 100644 index 966982fd5c..0000000000 --- a/ql/src/test/queries/clientnegative/nopart_load.q +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE TABLE nopart_load(a STRING, b STRING) PARTITIONED BY (ds STRING); - -load data local inpath '../../data/files/kv1.txt' overwrite into table nopart_load ; - diff --git a/ql/src/test/queries/clientpositive/load_data_using_job.q b/ql/src/test/queries/clientpositive/load_data_using_job.q new file mode 100644 index 0000000000..847c78f64c --- /dev/null +++ b/ql/src/test/queries/clientpositive/load_data_using_job.q @@ -0,0 +1,69 @@ +set hive.stats.column.autogather=false; +set hive.strict.checks.bucketing=false; + +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.auto.convert.join=true; +set hive.join.emit.interval=2; +set hive.auto.convert.join.noconditionaltask=true; +set hive.auto.convert.join.noconditionaltask.size=10000; +set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ; + +-- Single partition +-- Regular load happens. +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE; +explain load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08'); +load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08'); +select * from srcbucket_mapjoin; + +drop table srcbucket_mapjoin; + +-- Triggers a Tez job as partition info is missing from load data. +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Multi partitions +-- Triggers a Tez job as partition info is missing from load data. +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Multi partitions and directory with files (no sub dirs) +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Bucketing +CREATE TABLE srcbucket_mapjoin(key int, value string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Single partition and bucketing +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Multiple partitions and bucketing +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; + +-- Multiple partitions, bucketing, and directory with files (no sub dirs) +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE; +explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin; +load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin; +select * from srcbucket_mapjoin; +drop table srcbucket_mapjoin; \ No newline at end of file diff --git a/ql/src/test/results/clientnegative/load_part_nospec.q.out b/ql/src/test/results/clientnegative/load_part_nospec.q.out deleted file mode 100644 index bebaf92311..0000000000 --- a/ql/src/test/results/clientnegative/load_part_nospec.q.out +++ /dev/null @@ -1,9 +0,0 @@ -PREHOOK: query: create table hive_test_src ( col1 string ) partitioned by (pcol1 string) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@hive_test_src -POSTHOOK: query: create table hive_test_src ( col1 string ) partitioned by (pcol1 string) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@hive_test_src -FAILED: SemanticException [Error 10062]: Need to specify partition columns because the destination table is partitioned diff --git a/ql/src/test/results/clientnegative/nopart_load.q.out b/ql/src/test/results/clientnegative/nopart_load.q.out deleted file mode 100644 index 881514640c..0000000000 --- a/ql/src/test/results/clientnegative/nopart_load.q.out +++ /dev/null @@ -1,9 +0,0 @@ -PREHOOK: query: CREATE TABLE nopart_load(a STRING, b STRING) PARTITIONED BY (ds STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@nopart_load -POSTHOOK: query: CREATE TABLE nopart_load(a STRING, b STRING) PARTITIONED BY (ds STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@nopart_load -FAILED: SemanticException [Error 10062]: Need to specify partition columns because the destination table is partitioned diff --git a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out new file mode 100644 index 0000000000..40badf8587 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out @@ -0,0 +1,1954 @@ +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +POSTHOOK: query: explain load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-1 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2008-04-08 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-1 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +165 val_165 2008-04-08 +484 val_484 2008-04-08 +150 val_150 2008-04-08 +224 val_224 2008-04-08 +66 val_66 2008-04-08 +213 val_213 2008-04-08 +374 val_374 2008-04-08 +495 val_495 2008-04-08 +37 val_37 2008-04-08 +327 val_327 2008-04-08 +15 val_15 2008-04-08 +338 val_338 2008-04-08 +459 val_459 2008-04-08 +466 val_466 2008-04-08 +396 val_396 2008-04-08 +309 val_309 2008-04-08 +367 val_367 2008-04-08 +0 val_0 2008-04-08 +455 val_455 2008-04-08 +316 val_316 2008-04-08 +345 val_345 2008-04-08 +129 val_129 2008-04-08 +378 val_378 2008-04-08 +4 val_4 2008-04-08 +356 val_356 2008-04-08 +169 val_169 2008-04-08 +125 val_125 2008-04-08 +437 val_437 2008-04-08 +286 val_286 2008-04-08 +187 val_187 2008-04-08 +176 val_176 2008-04-08 +459 val_459 2008-04-08 +51 val_51 2008-04-08 +103 val_103 2008-04-08 +239 val_239 2008-04-08 +213 val_213 2008-04-08 +176 val_176 2008-04-08 +275 val_275 2008-04-08 +260 val_260 2008-04-08 +404 val_404 2008-04-08 +217 val_217 2008-04-08 +84 val_84 2008-04-08 +466 val_466 2008-04-08 +8 val_8 2008-04-08 +411 val_411 2008-04-08 +172 val_172 2008-04-08 +129 val_129 2008-04-08 +158 val_158 2008-04-08 +0 val_0 2008-04-08 +26 val_26 2008-04-08 +165 val_165 2008-04-08 +327 val_327 2008-04-08 +51 val_51 2008-04-08 +404 val_404 2008-04-08 +95 val_95 2008-04-08 +282 val_282 2008-04-08 +187 val_187 2008-04-08 +316 val_316 2008-04-08 +169 val_169 2008-04-08 +77 val_77 2008-04-08 +0 val_0 2008-04-08 +118 val_118 2008-04-08 +282 val_282 2008-04-08 +419 val_419 2008-04-08 +15 val_15 2008-04-08 +118 val_118 2008-04-08 +19 val_19 2008-04-08 +224 val_224 2008-04-08 +309 val_309 2008-04-08 +389 val_389 2008-04-08 +327 val_327 2008-04-08 +242 val_242 2008-04-08 +392 val_392 2008-04-08 +242 val_242 2008-04-08 +396 val_396 2008-04-08 +95 val_95 2008-04-08 +11 val_11 2008-04-08 +143 val_143 2008-04-08 +228 val_228 2008-04-08 +33 val_33 2008-04-08 +103 val_103 2008-04-08 +367 val_367 2008-04-08 +239 val_239 2008-04-08 +480 val_480 2008-04-08 +202 val_202 2008-04-08 +316 val_316 2008-04-08 +235 val_235 2008-04-08 +80 val_80 2008-04-08 +44 val_44 2008-04-08 +466 val_466 2008-04-08 +257 val_257 2008-04-08 +190 val_190 2008-04-08 +114 val_114 2008-04-08 +396 val_396 2008-04-08 +217 val_217 2008-04-08 +125 val_125 2008-04-08 +187 val_187 2008-04-08 +480 val_480 2008-04-08 +491 val_491 2008-04-08 +305 val_305 2008-04-08 +444 val_444 2008-04-08 +169 val_169 2008-04-08 +323 val_323 2008-04-08 +480 val_480 2008-04-08 +136 val_136 2008-04-08 +172 val_172 2008-04-08 +462 val_462 2008-04-08 +26 val_26 2008-04-08 +462 val_462 2008-04-08 +341 val_341 2008-04-08 +183 val_183 2008-04-08 +84 val_84 2008-04-08 +37 val_37 2008-04-08 +448 val_448 2008-04-08 +194 val_194 2008-04-08 +477 val_477 2008-04-08 +169 val_169 2008-04-08 +400 val_400 2008-04-08 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + Execution mode: vectorized, llap + LLAP IO: no inputs + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +165 val_165 2008-04-08 +484 val_484 2008-04-08 +150 val_150 2008-04-08 +224 val_224 2008-04-08 +66 val_66 2008-04-08 +213 val_213 2008-04-08 +374 val_374 2008-04-08 +495 val_495 2008-04-08 +37 val_37 2008-04-08 +327 val_327 2008-04-08 +15 val_15 2008-04-08 +338 val_338 2008-04-08 +459 val_459 2008-04-08 +466 val_466 2008-04-08 +396 val_396 2008-04-08 +309 val_309 2008-04-08 +367 val_367 2008-04-08 +0 val_0 2008-04-08 +455 val_455 2008-04-08 +316 val_316 2008-04-08 +345 val_345 2008-04-08 +129 val_129 2008-04-08 +378 val_378 2008-04-08 +4 val_4 2008-04-08 +356 val_356 2008-04-08 +169 val_169 2008-04-08 +125 val_125 2008-04-08 +437 val_437 2008-04-08 +286 val_286 2008-04-08 +187 val_187 2008-04-08 +176 val_176 2008-04-08 +459 val_459 2008-04-08 +51 val_51 2008-04-08 +103 val_103 2008-04-08 +239 val_239 2008-04-08 +213 val_213 2008-04-08 +176 val_176 2008-04-08 +275 val_275 2008-04-08 +260 val_260 2008-04-08 +404 val_404 2008-04-08 +217 val_217 2008-04-08 +84 val_84 2008-04-08 +466 val_466 2008-04-08 +8 val_8 2008-04-08 +411 val_411 2008-04-08 +172 val_172 2008-04-08 +129 val_129 2008-04-08 +158 val_158 2008-04-08 +0 val_0 2008-04-08 +26 val_26 2008-04-08 +165 val_165 2008-04-08 +327 val_327 2008-04-08 +51 val_51 2008-04-08 +404 val_404 2008-04-08 +95 val_95 2008-04-08 +282 val_282 2008-04-08 +187 val_187 2008-04-08 +316 val_316 2008-04-08 +169 val_169 2008-04-08 +77 val_77 2008-04-08 +0 val_0 2008-04-08 +118 val_118 2008-04-08 +282 val_282 2008-04-08 +419 val_419 2008-04-08 +15 val_15 2008-04-08 +118 val_118 2008-04-08 +19 val_19 2008-04-08 +224 val_224 2008-04-08 +309 val_309 2008-04-08 +389 val_389 2008-04-08 +327 val_327 2008-04-08 +242 val_242 2008-04-08 +392 val_392 2008-04-08 +242 val_242 2008-04-08 +396 val_396 2008-04-08 +95 val_95 2008-04-08 +11 val_11 2008-04-08 +143 val_143 2008-04-08 +228 val_228 2008-04-08 +33 val_33 2008-04-08 +103 val_103 2008-04-08 +367 val_367 2008-04-08 +239 val_239 2008-04-08 +480 val_480 2008-04-08 +202 val_202 2008-04-08 +316 val_316 2008-04-08 +235 val_235 2008-04-08 +80 val_80 2008-04-08 +44 val_44 2008-04-08 +466 val_466 2008-04-08 +257 val_257 2008-04-08 +190 val_190 2008-04-08 +114 val_114 2008-04-08 +396 val_396 2008-04-08 +217 val_217 2008-04-08 +125 val_125 2008-04-08 +187 val_187 2008-04-08 +480 val_480 2008-04-08 +491 val_491 2008-04-08 +305 val_305 2008-04-08 +444 val_444 2008-04-08 +169 val_169 2008-04-08 +323 val_323 2008-04-08 +480 val_480 2008-04-08 +136 val_136 2008-04-08 +172 val_172 2008-04-08 +462 val_462 2008-04-08 +26 val_26 2008-04-08 +462 val_462 2008-04-08 +341 val_341 2008-04-08 +183 val_183 2008-04-08 +84 val_84 2008-04-08 +37 val_37 2008-04-08 +448 val_448 2008-04-08 +194 val_194 2008-04-08 +477 val_477 2008-04-08 +169 val_169 2008-04-08 +400 val_400 2008-04-08 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + Execution mode: vectorized, llap + LLAP IO: no inputs + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + Execution mode: vectorized, llap + LLAP IO: no inputs + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/bucketing.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Lineage: srcbucket_mapjoin.key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin.value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +15 val_15 +15 val_15 +80 val_80 +95 val_95 +95 val_95 +125 val_125 +125 val_125 +150 val_150 +165 val_165 +165 val_165 +190 val_190 +235 val_235 +260 val_260 +275 val_275 +305 val_305 +345 val_345 +400 val_400 +455 val_455 +480 val_480 +480 val_480 +480 val_480 +495 val_495 +11 val_11 +26 val_26 +26 val_26 +51 val_51 +51 val_51 +66 val_66 +136 val_136 +176 val_176 +176 val_176 +286 val_286 +316 val_316 +316 val_316 +316 val_316 +341 val_341 +356 val_356 +396 val_396 +396 val_396 +396 val_396 +411 val_411 +466 val_466 +466 val_466 +466 val_466 +491 val_491 +37 val_37 +37 val_37 +77 val_77 +172 val_172 +172 val_172 +187 val_187 +187 val_187 +187 val_187 +202 val_202 +217 val_217 +217 val_217 +242 val_242 +242 val_242 +257 val_257 +282 val_282 +282 val_282 +327 val_327 +327 val_327 +327 val_327 +367 val_367 +367 val_367 +392 val_392 +437 val_437 +462 val_462 +462 val_462 +477 val_477 +8 val_8 +33 val_33 +103 val_103 +103 val_103 +118 val_118 +118 val_118 +143 val_143 +158 val_158 +183 val_183 +213 val_213 +213 val_213 +228 val_228 +323 val_323 +338 val_338 +378 val_378 +448 val_448 +4 val_4 +19 val_19 +44 val_44 +84 val_84 +84 val_84 +114 val_114 +129 val_129 +129 val_129 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +194 val_194 +224 val_224 +224 val_224 +239 val_239 +239 val_239 +309 val_309 +309 val_309 +374 val_374 +389 val_389 +404 val_404 +404 val_404 +419 val_419 +444 val_444 +459 val_459 +459 val_459 +484 val_484 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +#### A masked pattern was here #### +0 val_0 2008-04-08 +0 val_0 2008-04-08 +0 val_0 2008-04-08 +15 val_15 2008-04-08 +15 val_15 2008-04-08 +80 val_80 2008-04-08 +95 val_95 2008-04-08 +95 val_95 2008-04-08 +125 val_125 2008-04-08 +125 val_125 2008-04-08 +150 val_150 2008-04-08 +165 val_165 2008-04-08 +165 val_165 2008-04-08 +190 val_190 2008-04-08 +235 val_235 2008-04-08 +260 val_260 2008-04-08 +275 val_275 2008-04-08 +305 val_305 2008-04-08 +345 val_345 2008-04-08 +400 val_400 2008-04-08 +455 val_455 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +495 val_495 2008-04-08 +11 val_11 2008-04-08 +26 val_26 2008-04-08 +26 val_26 2008-04-08 +51 val_51 2008-04-08 +51 val_51 2008-04-08 +66 val_66 2008-04-08 +136 val_136 2008-04-08 +176 val_176 2008-04-08 +176 val_176 2008-04-08 +286 val_286 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +341 val_341 2008-04-08 +356 val_356 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +411 val_411 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +491 val_491 2008-04-08 +37 val_37 2008-04-08 +37 val_37 2008-04-08 +77 val_77 2008-04-08 +172 val_172 2008-04-08 +172 val_172 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +202 val_202 2008-04-08 +217 val_217 2008-04-08 +217 val_217 2008-04-08 +242 val_242 2008-04-08 +242 val_242 2008-04-08 +257 val_257 2008-04-08 +282 val_282 2008-04-08 +282 val_282 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +367 val_367 2008-04-08 +367 val_367 2008-04-08 +392 val_392 2008-04-08 +437 val_437 2008-04-08 +462 val_462 2008-04-08 +462 val_462 2008-04-08 +477 val_477 2008-04-08 +8 val_8 2008-04-08 +33 val_33 2008-04-08 +103 val_103 2008-04-08 +103 val_103 2008-04-08 +118 val_118 2008-04-08 +118 val_118 2008-04-08 +143 val_143 2008-04-08 +158 val_158 2008-04-08 +183 val_183 2008-04-08 +213 val_213 2008-04-08 +213 val_213 2008-04-08 +228 val_228 2008-04-08 +323 val_323 2008-04-08 +338 val_338 2008-04-08 +378 val_378 2008-04-08 +448 val_448 2008-04-08 +4 val_4 2008-04-08 +19 val_19 2008-04-08 +44 val_44 2008-04-08 +84 val_84 2008-04-08 +84 val_84 2008-04-08 +114 val_114 2008-04-08 +129 val_129 2008-04-08 +129 val_129 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +194 val_194 2008-04-08 +224 val_224 2008-04-08 +224 val_224 2008-04-08 +239 val_239 2008-04-08 +239 val_239 2008-04-08 +309 val_309 2008-04-08 +309 val_309 2008-04-08 +374 val_374 2008-04-08 +389 val_389 2008-04-08 +404 val_404 2008-04-08 +404 val_404 2008-04-08 +419 val_419 2008-04-08 +444 val_444 2008-04-08 +459 val_459 2008-04-08 +459 val_459 2008-04-08 +484 val_484 2008-04-08 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +0 val_0 2008-04-08 0 +80 val_80 2008-04-08 0 +95 val_95 2008-04-08 0 +125 val_125 2008-04-08 0 +190 val_190 2008-04-08 0 +275 val_275 2008-04-08 0 +305 val_305 2008-04-08 0 +400 val_400 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +495 val_495 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +341 val_341 2008-04-08 0 +396 val_396 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +77 val_77 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +187 val_187 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +282 val_282 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +367 val_367 2008-04-08 0 +437 val_437 2008-04-08 0 +477 val_477 2008-04-08 0 +8 val_8 2008-04-08 0 +33 val_33 2008-04-08 0 +103 val_103 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +143 val_143 2008-04-08 0 +158 val_158 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +338 val_338 2008-04-08 0 +448 val_448 2008-04-08 0 +4 val_4 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +129 val_129 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +309 val_309 2008-04-08 0 +389 val_389 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +419 val_419 2008-04-08 0 +459 val_459 2008-04-08 0 +484 val_484 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +95 val_95 2008-04-08 1 +125 val_125 2008-04-08 1 +150 val_150 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +235 val_235 2008-04-08 1 +260 val_260 2008-04-08 1 +345 val_345 2008-04-08 1 +455 val_455 2008-04-08 1 +11 val_11 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +136 val_136 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +286 val_286 2008-04-08 1 +356 val_356 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +411 val_411 2008-04-08 1 +466 val_466 2008-04-08 1 +491 val_491 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +202 val_202 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +257 val_257 2008-04-08 1 +282 val_282 2008-04-08 1 +327 val_327 2008-04-08 1 +367 val_367 2008-04-08 1 +392 val_392 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +103 val_103 2008-04-08 1 +183 val_183 2008-04-08 1 +228 val_228 2008-04-08 1 +323 val_323 2008-04-08 1 +378 val_378 2008-04-08 1 +19 val_19 2008-04-08 1 +44 val_44 2008-04-08 1 +114 val_114 2008-04-08 1 +129 val_129 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +194 val_194 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +309 val_309 2008-04-08 1 +374 val_374 2008-04-08 1 +444 val_444 2008-04-08 1 +459 val_459 2008-04-08 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin__temp_table_for_load_data__ + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__ +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin__TEMP_TABLE_FOR_LOAD_DATA__)srcbucket_mapjoin__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=0 +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +0 val_0 2008-04-08 0 +0 val_0 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 0 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +495 val_495 2008-04-08 0 +495 val_495 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +437 val_437 2008-04-08 0 +437 val_437 2008-04-08 0 +477 val_477 2008-04-08 0 +477 val_477 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +158 val_158 2008-04-08 0 +158 val_158 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +338 val_338 2008-04-08 0 +338 val_338 2008-04-08 0 +448 val_448 2008-04-08 0 +448 val_448 2008-04-08 0 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 +389 val_389 2008-04-08 0 +389 val_389 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +419 val_419 2008-04-08 0 +419 val_419 2008-04-08 0 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +150 val_150 2008-04-08 1 +150 val_150 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +235 val_235 2008-04-08 1 +235 val_235 2008-04-08 1 +260 val_260 2008-04-08 1 +260 val_260 2008-04-08 1 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 +11 val_11 2008-04-08 1 +11 val_11 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +136 val_136 2008-04-08 1 +136 val_136 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 +491 val_491 2008-04-08 1 +491 val_491 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +202 val_202 2008-04-08 1 +202 val_202 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +257 val_257 2008-04-08 1 +257 val_257 2008-04-08 1 +282 val_282 2008-04-08 1 +282 val_282 2008-04-08 1 +327 val_327 2008-04-08 1 +327 val_327 2008-04-08 1 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +392 val_392 2008-04-08 1 +392 val_392 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 +183 val_183 2008-04-08 1 +183 val_183 2008-04-08 1 +228 val_228 2008-04-08 1 +228 val_228 2008-04-08 1 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 +19 val_19 2008-04-08 1 +19 val_19 2008-04-08 1 +44 val_44 2008-04-08 1 +44 val_44 2008-04-08 1 +114 val_114 2008-04-08 1 +114 val_114 2008-04-08 1 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +309 val_309 2008-04-08 1 +309 val_309 2008-04-08 1 +374 val_374 2008-04-08 1 +374 val_374 2008-04-08 1 +444 val_444 2008-04-08 1 +444 val_444 2008-04-08 1 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin