commit 4d800d18ad9afc013d044b69453db91743fe8f78 Author: Janaki Lahorani Date: Fri Aug 11 09:25:31 2017 -0700 Fix HIVE-17225. Move the partition pruning to a separate spark job if the target is not a map join. The check to determine whether the job to determine the list of partitions to scan will be pushed to separate spark job was based on whether there is a hash table sink operator, with the implicit assumption that the target of pruning sink operator will be same as the target for hash table sink operator. When 3 or more tables are joined, it is possible to generate a plan where the target of partition pruning sink operator is another hash table sink operator and not the map join. In this case both source and target will run as concurrent map jobs, and can result in File Not Found Exception. The fix is to recognize this and move the work with pruning sink operator to a separate parent spark job. diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java index a3ec990483eb1d07617c051edb16f9cf64670b07..64a0e455f2ff95a1e77cefa988cd9aed563b194c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.ConditionalResolver; import org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin; @@ -155,25 +156,31 @@ private void moveWork(SparkWork sparkWork, BaseWork work, SparkWork targetWork) } } + SparkWork parentWork = null; + if (!containsOp(work, MapJoinOperator.class)) { + // This is not Map Join. If parents are there, they will be because of Partition + // Pruning Sink Operator. for (BaseWork parent : parentWorks) { - moveWork(sparkWork, parent, targetWork); + if (containsOp(parent, SparkPartitionPruningSinkOperator.class)) { + // Parent Work is created first time a Partition Pruning Sink is seen. If there aren't + // partition pruning sink, there won't be a need to create parent. + if (parentWork == null) parentWork = createParentWork(targetWork, sparkWork, work); + moveWork(sparkWork, parent, parentWork); + } + else { + moveWork(sparkWork, parent, targetWork); + } } } else { - // Create a new SparkWork for all the small tables of this work - SparkWork parentWork = - new SparkWork(physicalContext.conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); - // copy cloneToWork to ensure RDD cache still works - parentWork.setCloneToWork(sparkWork.getCloneToWork()); - - dependencyGraph.get(targetWork).add(parentWork); - dependencyGraph.put(parentWork, new ArrayList()); - - // this work is now moved to the parentWork, thus we should - // update this information in sparkWorkMap - sparkWorkMap.put(work, parentWork); + // This is Map Join. The parents can either be because of Hash Table Sink and possibly + // Partition Pruning Sink. Definitely there has to be Hash Table Sink. Create parent + // always. + parentWork = createParentWork(targetWork, sparkWork, work); + for (BaseWork parent : parentWorks) { - if (containsOp(parent, SparkHashTableSinkOperator.class)) { + if (containsOp(parent, SparkHashTableSinkOperator.class) || + containsOp(parent, SparkPartitionPruningSinkOperator.class)) { moveWork(sparkWork, parent, parentWork); } else { moveWork(sparkWork, parent, targetWork); @@ -182,6 +189,24 @@ private void moveWork(SparkWork sparkWork, BaseWork work, SparkWork targetWork) } } + // Create a parent work to which some base works will get moved to. + private SparkWork createParentWork(SparkWork targetWork, SparkWork sparkWork, BaseWork work) { + // Create a new SparkWork + SparkWork parentWork = + new SparkWork(physicalContext.conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); + + // copy cloneToWork to ensure RDD cache still works + parentWork.setCloneToWork(sparkWork.getCloneToWork()); + + dependencyGraph.get(targetWork).add(parentWork); + dependencyGraph.put(parentWork, new ArrayList()); + + // remove of base works happens separately. Record information to facilitate removal. + sparkWorkMap.put(work, parentWork); + + return parentWork; + } + private void generateLocalWork(SparkTask originalTask) { SparkWork originalWork = originalTask.getWork(); Collection allBaseWorks = originalWork.getAllWork(); @@ -332,6 +357,9 @@ private void processCurrentTask(SparkTask sparkTask, ConditionalTask conditional // Generate MapredLocalWorks for MJ and HTS generateLocalWork(sparkTask); + // Move the dpp edges stored in to do list to edge properties. + sparkWork.moveToDo2EdgeProperty(); + dependencyGraph.put(sparkWork, new ArrayList()); Set leaves = sparkWork.getLeaves(); for (BaseWork leaf : leaves) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 4f8ba6ffd3d71bda23402711e66cdfdfb91c41c3..f365d2eb65d17214ff0c842104e48cca86b4bbe3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -506,6 +506,20 @@ public void processPartitionPruningSink(GenSparkProcContext context, } List keys = targetWork.getEventSourcePartKeyExprMap().get(sourceId); keys.add(desc.getPartKey()); + + // Table Scan with Pruning Sink Operators that are not part of mapjoins are cloned + // because these are executed using a separate spark job. + if (!pruningSink.isCloned()) { + // Partition Pruning Sink Operator is under the Hash Table Sink Operator + SparkWork sparkWork = context.currentTask.getWork(); + if (sparkWork.getEdgeProperty(sourceWork, targetWork) == null) { + // The target job of Partition Pruning Sink is not the Consumer of Hash Table Sink. + // The dependency needs to be added after map join processing. Record it for + // processing later. + sparkWork.putToDoEdgeProperty(sourceWork, targetWork, + new SparkEdgeProperty(SparkEdgeProperty.SHUFFLE_NONE)); + } + } } public static SparkEdgeProperty getEdgeProperty(HiveConf conf, ReduceSinkOperator reduceSink, diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java index e3146cf06a8f8253e3f333a1c3dd8f934a8077cd..64738e982a89a9acb92b63d58b9d687bfa1ca548 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkPartitionPruningSinkOperator.java @@ -56,6 +56,8 @@ protected transient DataOutputBuffer buffer; protected static final Logger LOG = LoggerFactory.getLogger(SparkPartitionPruningSinkOperator.class); + private boolean cloned = false; + /** Kryo ctor. */ @VisibleForTesting public SparkPartitionPruningSinkOperator() { @@ -144,6 +146,9 @@ public boolean isWithMapjoin() { return branchingOp; } + public void setCloned() {this.cloned = true;} + public boolean isCloned() {return this.cloned;} + private void flushToFile() throws IOException { // write an intermediate file to the specified path // the format of the path is: tmpPath/targetWorkId/sourceWorkId/randInt diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java index db0f54e87d6ffd61c9079805ad9e11343533e1ec..b4011ddd09afa81c99c944a4e6525aeb11064462 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java @@ -138,6 +138,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, ((SparkPartitionPruningSinkOperator) clonedPruningSinkOp).getConf().setTableScan(oldsinkOp.getConf().getTableScan()); context.pruningSinkSet.add(clonedPruningSinkOp); + ((SparkPartitionPruningSinkOperator) clonedPruningSinkOp).setCloned(); } return null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java index 9ca5544b491ac1b5a854657c66dee8ca538c18b3..faf59c058664756ec4012a149bb5db8a5eeef2ab 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java @@ -62,6 +62,10 @@ protected final Map, SparkEdgeProperty> edgeProperties = new HashMap, SparkEdgeProperty>(); + // These are edge properties that are to be moved to edgeProperties at a later time + protected Map, SparkEdgeProperty> toDoEdgeProperties = + new HashMap, SparkEdgeProperty>(); + private Map> requiredCounterPrefix; private Map cloneToWork; @@ -285,6 +289,91 @@ public void connect(BaseWork a, BaseWork b, SparkEdgeProperty edgeProp) { edgeProperties.put(workPair, edgeProp); } + public void putToDoEdgeProperty(BaseWork a, BaseWork b, SparkEdgeProperty toDoEdgeProperty) { + ImmutablePair workPair = new ImmutablePair(a, b); + toDoEdgeProperties.put(workPair, toDoEdgeProperty); + } + + // sourcework and targetwork will get connected. recursively remove edges that exist between + // sourcework and children of target + private void disconnectMultiConnections(BaseWork sourceWork, BaseWork targetWork, + Map, SparkEdgeProperty> remEdgeProperties) { + List childWorks = getChildren(targetWork); + for (BaseWork child : childWorks) { + if (getEdgeProperty(sourceWork, child) != null) { + remEdgeProperties.put(new ImmutablePair(sourceWork, child), + getEdgeProperty(sourceWork, child)); + } + disconnectMultiConnections(sourceWork, child, remEdgeProperties); + } + } + + private void disconnectTodoMultiConnections(BaseWork sourceWork, BaseWork targetWork, + Map, SparkEdgeProperty> remToDoEdgeProperties) { + for (Map.Entry, SparkEdgeProperty> toDoEdgeProperty : + toDoEdgeProperties.entrySet()) { + if (toDoEdgeProperty.getKey().getLeft() == targetWork) { + BaseWork child = toDoEdgeProperty.getKey().getRight(); + + if (toDoEdgeProperties.get(new ImmutablePair(sourceWork, child)) + != null) { + remToDoEdgeProperties.put(new ImmutablePair(sourceWork, child), + toDoEdgeProperties.get(new ImmutablePair(sourceWork, child))); + } + disconnectTodoMultiConnections(sourceWork, child, remToDoEdgeProperties); + } + } + } + + public void moveToDo2EdgeProperty() { + if (toDoEdgeProperties.isEmpty()) + return; + + // To track edges that need to be removed + Map, SparkEdgeProperty> remEdgeProperties = new HashMap<>(); + + // Find all redundant edges from to do edge properties + for (Map.Entry, SparkEdgeProperty> toDoEdgeProperty : + toDoEdgeProperties.entrySet()) { + disconnectTodoMultiConnections(toDoEdgeProperty.getKey().getLeft(), + toDoEdgeProperty.getKey().getRight(), remEdgeProperties); + } + + // Remove redundant edges from to do edge properties + for (Map.Entry, SparkEdgeProperty> remToDoEdgeProperty : + remEdgeProperties.entrySet()) { + toDoEdgeProperties.remove(new ImmutablePair(remToDoEdgeProperty.getKey().getLeft(), + remToDoEdgeProperty.getKey().getRight())); + } + + // initialize remEdgeProperties - now it will be used to track edges from edge properties + remEdgeProperties = new HashMap<>(); + + // Moving from To Do list can cause multiple ways to go from a to b. Find redundant edges + // from edge properties caused because of moving edges from to do. + for (Map.Entry, SparkEdgeProperty> toDoEdgeProperty : + toDoEdgeProperties.entrySet()) { + disconnectMultiConnections(toDoEdgeProperty.getKey().getLeft(), + toDoEdgeProperty.getKey().getRight(), remEdgeProperties); + } + + // Remove redundant edges from edge properties because of moving edges from to do + for (Map.Entry, SparkEdgeProperty> remEdgeProperty : + remEdgeProperties.entrySet()) { + disconnect(remEdgeProperty.getKey().getLeft(), remEdgeProperty.getKey().getRight()); + } + + // Add the to do edges to edge properties + for (Map.Entry, SparkEdgeProperty> toDoEdgeProperty : + toDoEdgeProperties.entrySet()) { + connect(toDoEdgeProperty.getKey().getLeft(), toDoEdgeProperty.getKey().getRight(), + toDoEdgeProperty.getValue()); + } + + // cleanup to do list + toDoEdgeProperties = new HashMap, SparkEdgeProperty>(); + } + /* * Dependency is a class used for explain */ diff --git ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_recursive_mapjoin.q ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_recursive_mapjoin.q new file mode 100644 index 0000000000000000000000000000000000000000..11a05f12f1f69e1871352074216b46ff9aee6c5a --- /dev/null +++ ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_recursive_mapjoin.q @@ -0,0 +1,235 @@ +SET hive.spark.dynamic.partition.pruning=true; +SET hive.auto.convert.join=true; +SET hive.strict.checks.cartesian.product=false; + +CREATE TABLE part_table1 (col int) PARTITIONED BY (part1_col int); +CREATE TABLE part_table2 (col int) PARTITIONED BY (part2_col int); +CREATE TABLE part_table3 (col int) PARTITIONED BY (part3_col int); +CREATE TABLE part_table4 (col int) PARTITIONED BY (part4_col int); +CREATE TABLE part_table5 (col int) PARTITIONED BY (part5_col int); + +CREATE TABLE reg_table (col int); + +ALTER TABLE part_table1 ADD PARTITION (part1_col = 1); + +ALTER TABLE part_table2 ADD PARTITION (part2_col = 1); +ALTER TABLE part_table2 ADD PARTITION (part2_col = 2); + +ALTER TABLE part_table3 ADD PARTITION (part3_col = 1); +ALTER TABLE part_table3 ADD PARTITION (part3_col = 2); +ALTER TABLE part_table3 ADD PARTITION (part3_col = 3); + +ALTER TABLE part_table4 ADD PARTITION (part4_col = 1); +ALTER TABLE part_table4 ADD PARTITION (part4_col = 2); +ALTER TABLE part_table4 ADD PARTITION (part4_col = 3); +ALTER TABLE part_table4 ADD PARTITION (part4_col = 4); + +ALTER TABLE part_table5 ADD PARTITION (part5_col = 1); +ALTER TABLE part_table5 ADD PARTITION (part5_col = 2); +ALTER TABLE part_table5 ADD PARTITION (part5_col = 3); +ALTER TABLE part_table5 ADD PARTITION (part5_col = 4); +ALTER TABLE part_table5 ADD PARTITION (part5_col = 5); + +INSERT INTO TABLE part_table1 PARTITION (part1_col = 1) VALUES (1); + +INSERT INTO TABLE part_table2 PARTITION (part2_col = 1) VALUES (1); +INSERT INTO TABLE part_table2 PARTITION (part2_col = 2) VALUES (2); + +INSERT INTO TABLE part_table3 PARTITION (part3_col = 1) VALUES (1); +INSERT INTO TABLE part_table3 PARTITION (part3_col = 2) VALUES (2); +INSERT INTO TABLE part_table3 PARTITION (part3_col = 3) VALUES (3); + +INSERT INTO TABLE part_table4 PARTITION (part4_col = 1) VALUES (1); +INSERT INTO TABLE part_table4 PARTITION (part4_col = 2) VALUES (2); +INSERT INTO TABLE part_table4 PARTITION (part4_col = 3) VALUES (3); +INSERT INTO TABLE part_table4 PARTITION (part4_col = 4) VALUES (4); + +INSERT INTO TABLE part_table5 PARTITION (part5_col = 1) VALUES (1); +INSERT INTO TABLE part_table5 PARTITION (part5_col = 2) VALUES (2); +INSERT INTO TABLE part_table5 PARTITION (part5_col = 3) VALUES (3); +INSERT INTO TABLE part_table5 PARTITION (part5_col = 4) VALUES (4); +INSERT INTO TABLE part_table5 PARTITION (part5_col = 5) VALUES (5); + +INSERT INTO table reg_table VALUES (1), (2), (3), (4), (5), (6); + +-- 3 table join pt2 pruned based on scan from pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col; + +-- 4 table join pt3 pruned based on pt2, pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col; + +-- 5 table join pt4 pruned based on pt3, pt3 pruned based on pt2, pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col; + +-- 6 table join pt5 pruned based on pt4, pt4 pruned based on pt3, pt3 pruned based on pt2, +-- pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col; + +-- Same tests with Partition Pruning turned off +set hive.spark.dynamic.partition.pruning=false; + +-- 3 table join pt2 pruned based on scan from pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col; + +-- 4 table join pt3 pruned based on pt2, pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col; + +-- 5 table join pt4 pruned based on pt3, pt3 pruned based on pt2, pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col; + +-- 6 table join pt5 pruned based on pt4, pt4 pruned based on pt3, pt3 pruned based on pt2, +-- pt2 pruned based on pt1 +explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col; + +SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col; + +-- Cleanup +DROP TABLE part_table1; +DROP TABLE part_table2; +DROP TABLE part_table3; +DROP TABLE part_table4; +DROP TABLE part_table5; + +DROP TABLE reg_table; diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_recursive_mapjoin.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_recursive_mapjoin.q.out new file mode 100644 index 0000000000000000000000000000000000000000..4d6e365ed4846b1553eee6999ed1bbc529842309 --- /dev/null +++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_recursive_mapjoin.q.out @@ -0,0 +1,2189 @@ +PREHOOK: query: CREATE TABLE part_table1 (col int) PARTITIONED BY (part1_col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_table1 +POSTHOOK: query: CREATE TABLE part_table1 (col int) PARTITIONED BY (part1_col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_table1 +PREHOOK: query: CREATE TABLE part_table2 (col int) PARTITIONED BY (part2_col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_table2 +POSTHOOK: query: CREATE TABLE part_table2 (col int) PARTITIONED BY (part2_col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_table2 +PREHOOK: query: CREATE TABLE part_table3 (col int) PARTITIONED BY (part3_col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_table3 +POSTHOOK: query: CREATE TABLE part_table3 (col int) PARTITIONED BY (part3_col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_table3 +PREHOOK: query: CREATE TABLE part_table4 (col int) PARTITIONED BY (part4_col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_table4 +POSTHOOK: query: CREATE TABLE part_table4 (col int) PARTITIONED BY (part4_col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_table4 +PREHOOK: query: CREATE TABLE part_table5 (col int) PARTITIONED BY (part5_col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_table5 +POSTHOOK: query: CREATE TABLE part_table5 (col int) PARTITIONED BY (part5_col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_table5 +PREHOOK: query: CREATE TABLE reg_table (col int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@reg_table +POSTHOOK: query: CREATE TABLE reg_table (col int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@reg_table +PREHOOK: query: ALTER TABLE part_table1 ADD PARTITION (part1_col = 1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table1 +POSTHOOK: query: ALTER TABLE part_table1 ADD PARTITION (part1_col = 1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table1 +POSTHOOK: Output: default@part_table1@part1_col=1 +PREHOOK: query: ALTER TABLE part_table2 ADD PARTITION (part2_col = 1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table2 +POSTHOOK: query: ALTER TABLE part_table2 ADD PARTITION (part2_col = 1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table2 +POSTHOOK: Output: default@part_table2@part2_col=1 +PREHOOK: query: ALTER TABLE part_table2 ADD PARTITION (part2_col = 2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table2 +POSTHOOK: query: ALTER TABLE part_table2 ADD PARTITION (part2_col = 2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table2 +POSTHOOK: Output: default@part_table2@part2_col=2 +PREHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table3 +POSTHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table3 +POSTHOOK: Output: default@part_table3@part3_col=1 +PREHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table3 +POSTHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table3 +POSTHOOK: Output: default@part_table3@part3_col=2 +PREHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 3) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table3 +POSTHOOK: query: ALTER TABLE part_table3 ADD PARTITION (part3_col = 3) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table3 +POSTHOOK: Output: default@part_table3@part3_col=3 +PREHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table4 +POSTHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table4 +POSTHOOK: Output: default@part_table4@part4_col=1 +PREHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table4 +POSTHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table4 +POSTHOOK: Output: default@part_table4@part4_col=2 +PREHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 3) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table4 +POSTHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 3) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table4 +POSTHOOK: Output: default@part_table4@part4_col=3 +PREHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 4) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table4 +POSTHOOK: query: ALTER TABLE part_table4 ADD PARTITION (part4_col = 4) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table4 +POSTHOOK: Output: default@part_table4@part4_col=4 +PREHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table5 +POSTHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table5 +POSTHOOK: Output: default@part_table5@part5_col=1 +PREHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table5 +POSTHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table5 +POSTHOOK: Output: default@part_table5@part5_col=2 +PREHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 3) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table5 +POSTHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 3) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table5 +POSTHOOK: Output: default@part_table5@part5_col=3 +PREHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 4) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table5 +POSTHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 4) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table5 +POSTHOOK: Output: default@part_table5@part5_col=4 +PREHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 5) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@part_table5 +POSTHOOK: query: ALTER TABLE part_table5 ADD PARTITION (part5_col = 5) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@part_table5 +POSTHOOK: Output: default@part_table5@part5_col=5 +PREHOOK: query: INSERT INTO TABLE part_table1 PARTITION (part1_col = 1) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table1@part1_col=1 +POSTHOOK: query: INSERT INTO TABLE part_table1 PARTITION (part1_col = 1) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table1@part1_col=1 +POSTHOOK: Lineage: part_table1 PARTITION(part1_col=1).col EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table2 PARTITION (part2_col = 1) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table2@part2_col=1 +POSTHOOK: query: INSERT INTO TABLE part_table2 PARTITION (part2_col = 1) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table2@part2_col=1 +POSTHOOK: Lineage: part_table2 PARTITION(part2_col=1).col EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table2 PARTITION (part2_col = 2) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table2@part2_col=2 +POSTHOOK: query: INSERT INTO TABLE part_table2 PARTITION (part2_col = 2) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table2@part2_col=2 +POSTHOOK: Lineage: part_table2 PARTITION(part2_col=2).col EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 1) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table3@part3_col=1 +POSTHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 1) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table3@part3_col=1 +POSTHOOK: Lineage: part_table3 PARTITION(part3_col=1).col EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 2) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table3@part3_col=2 +POSTHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 2) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table3@part3_col=2 +POSTHOOK: Lineage: part_table3 PARTITION(part3_col=2).col EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 3) VALUES (3) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table3@part3_col=3 +POSTHOOK: query: INSERT INTO TABLE part_table3 PARTITION (part3_col = 3) VALUES (3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table3@part3_col=3 +POSTHOOK: Lineage: part_table3 PARTITION(part3_col=3).col EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 1) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table4@part4_col=1 +POSTHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 1) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table4@part4_col=1 +POSTHOOK: Lineage: part_table4 PARTITION(part4_col=1).col EXPRESSION [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 2) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table4@part4_col=2 +POSTHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 2) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table4@part4_col=2 +POSTHOOK: Lineage: part_table4 PARTITION(part4_col=2).col EXPRESSION [(values__tmp__table__8)values__tmp__table__8.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 3) VALUES (3) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table4@part4_col=3 +POSTHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 3) VALUES (3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table4@part4_col=3 +POSTHOOK: Lineage: part_table4 PARTITION(part4_col=3).col EXPRESSION [(values__tmp__table__9)values__tmp__table__9.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 4) VALUES (4) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table4@part4_col=4 +POSTHOOK: query: INSERT INTO TABLE part_table4 PARTITION (part4_col = 4) VALUES (4) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table4@part4_col=4 +POSTHOOK: Lineage: part_table4 PARTITION(part4_col=4).col EXPRESSION [(values__tmp__table__10)values__tmp__table__10.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 1) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table5@part5_col=1 +POSTHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 1) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table5@part5_col=1 +POSTHOOK: Lineage: part_table5 PARTITION(part5_col=1).col EXPRESSION [(values__tmp__table__11)values__tmp__table__11.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 2) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table5@part5_col=2 +POSTHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 2) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table5@part5_col=2 +POSTHOOK: Lineage: part_table5 PARTITION(part5_col=2).col EXPRESSION [(values__tmp__table__12)values__tmp__table__12.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 3) VALUES (3) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table5@part5_col=3 +POSTHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 3) VALUES (3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table5@part5_col=3 +POSTHOOK: Lineage: part_table5 PARTITION(part5_col=3).col EXPRESSION [(values__tmp__table__13)values__tmp__table__13.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 4) VALUES (4) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table5@part5_col=4 +POSTHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 4) VALUES (4) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table5@part5_col=4 +POSTHOOK: Lineage: part_table5 PARTITION(part5_col=4).col EXPRESSION [(values__tmp__table__14)values__tmp__table__14.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 5) VALUES (5) +PREHOOK: type: QUERY +PREHOOK: Output: default@part_table5@part5_col=5 +POSTHOOK: query: INSERT INTO TABLE part_table5 PARTITION (part5_col = 5) VALUES (5) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@part_table5@part5_col=5 +POSTHOOK: Lineage: part_table5 PARTITION(part5_col=5).col EXPRESSION [(values__tmp__table__15)values__tmp__table__15.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO table reg_table VALUES (1), (2), (3), (4), (5), (6) +PREHOOK: type: QUERY +PREHOOK: Output: default@reg_table +POSTHOOK: query: INSERT INTO table reg_table VALUES (1), (2), (3), (4), (5), (6) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@reg_table +POSTHOOK: Lineage: reg_table.col EXPRESSION [(values__tmp__table__16)values__tmp__table__16.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part2_col (int) + partition key expr: part2_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 2 + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + input vertices: + 0 Map 1 + 1 Map 2 + Statistics: Num rows: 13 Data size: 13 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 13 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-4 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part2_col (int) + partition key expr: part2_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 2 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Local Work: + Map Reduce Local Work + + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 + Stage-3 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-5 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part2_col (int) + partition key expr: part2_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 2 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Local Work: + Map Reduce Local Work + + Stage: Stage-4 + Spark +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Local Work: + Map Reduce Local Work + + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: pt4 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part4_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 5 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + 3 Map 4 + Statistics: Num rows: 26 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@part_table4 +PREHOOK: Input: default@part_table4@part4_col=1 +PREHOOK: Input: default@part_table4@part4_col=2 +PREHOOK: Input: default@part_table4@part4_col=3 +PREHOOK: Input: default@part_table4@part4_col=4 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@part_table4 +POSTHOOK: Input: default@part_table4@part4_col=1 +POSTHOOK: Input: default@part_table4@part4_col=2 +POSTHOOK: Input: default@part_table4@part4_col=3 +POSTHOOK: Input: default@part_table4@part4_col=4 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-5 depends on stages: Stage-6 + Stage-4 depends on stages: Stage-5 + Stage-3 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-6 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part2_col (int) + partition key expr: part2_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 2 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part5_col (int) + partition key expr: part5_col + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + target work: Map 5 + Local Work: + Map Reduce Local Work + + Stage: Stage-5 + Spark +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part3_col (int) + partition key expr: part3_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 3 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part5_col (int) + partition key expr: part5_col + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + target work: Map 5 + Local Work: + Map Reduce Local Work + + Stage: Stage-4 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part4_col (int) + partition key expr: part4_col + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + target work: Map 4 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part5_col (int) + partition key expr: part5_col + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + target work: Map 5 + Local Work: + Map Reduce Local Work + + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: pt4 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part4_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + Target column: part5_col (int) + partition key expr: part5_col + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + target work: Map 5 + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 5 + Map Operator Tree: + TableScan + alias: pt5 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part5_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 6 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + 3 Map 4 + 4 Map 5 + Statistics: Num rows: 33 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 33 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@part_table4 +PREHOOK: Input: default@part_table4@part4_col=1 +PREHOOK: Input: default@part_table4@part4_col=2 +PREHOOK: Input: default@part_table4@part4_col=3 +PREHOOK: Input: default@part_table4@part4_col=4 +PREHOOK: Input: default@part_table5 +PREHOOK: Input: default@part_table5@part5_col=1 +PREHOOK: Input: default@part_table5@part5_col=2 +PREHOOK: Input: default@part_table5@part5_col=3 +PREHOOK: Input: default@part_table5@part5_col=4 +PREHOOK: Input: default@part_table5@part5_col=5 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@part_table4 +POSTHOOK: Input: default@part_table4@part4_col=1 +POSTHOOK: Input: default@part_table4@part4_col=2 +POSTHOOK: Input: default@part_table4@part4_col=3 +POSTHOOK: Input: default@part_table4@part4_col=4 +POSTHOOK: Input: default@part_table5 +POSTHOOK: Input: default@part_table5@part5_col=1 +POSTHOOK: Input: default@part_table5@part5_col=2 +POSTHOOK: Input: default@part_table5@part5_col=3 +POSTHOOK: Input: default@part_table5@part5_col=4 +POSTHOOK: Input: default@part_table5@part5_col=5 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + input vertices: + 0 Map 1 + 1 Map 2 + Statistics: Num rows: 13 Data size: 13 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 13 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 4 + Map Operator Tree: + TableScan + alias: pt4 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part4_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 5 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + 3 Map 4 + Statistics: Num rows: 26 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@part_table4 +PREHOOK: Input: default@part_table4@part4_col=1 +PREHOOK: Input: default@part_table4@part4_col=2 +PREHOOK: Input: default@part_table4@part4_col=3 +PREHOOK: Input: default@part_table4@part4_col=4 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@part_table4 +POSTHOOK: Input: default@part_table4@part4_col=1 +POSTHOOK: Input: default@part_table4@part4_col=2 +POSTHOOK: Input: default@part_table4@part4_col=3 +POSTHOOK: Input: default@part_table4@part4_col=4 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 1 1 +PREHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col +PREHOOK: type: QUERY +POSTHOOK: query: explain SELECT * + FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt + WHERE rt.col = pt1.part1_col + AND pt2.part2_col = pt1.part1_col + AND pt3.part3_col = pt1.part1_col + AND pt4.part4_col = pt1.part1_col + AND pt5.part5_col = pt1.part1_col +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pt1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part1_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 2 + Map Operator Tree: + TableScan + alias: pt2 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part2_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 3 + Map Operator Tree: + TableScan + alias: pt3 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part3_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 4 + Map Operator Tree: + TableScan + alias: pt4 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part4_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + Map 5 + Map Operator Tree: + TableScan + alias: pt5 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int), part5_col (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 6 + Map Operator Tree: + TableScan + alias: rt + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: col is not null (type: boolean) + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + 2 _col1 (type: int) + 3 _col1 (type: int) + 4 _col1 (type: int) + 5 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + input vertices: + 0 Map 1 + 1 Map 2 + 2 Map 3 + 3 Map 4 + 4 Map 5 + Statistics: Num rows: 33 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 33 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col +PREHOOK: type: QUERY +PREHOOK: Input: default@part_table1 +PREHOOK: Input: default@part_table1@part1_col=1 +PREHOOK: Input: default@part_table2 +PREHOOK: Input: default@part_table2@part2_col=1 +PREHOOK: Input: default@part_table2@part2_col=2 +PREHOOK: Input: default@part_table3 +PREHOOK: Input: default@part_table3@part3_col=1 +PREHOOK: Input: default@part_table3@part3_col=2 +PREHOOK: Input: default@part_table3@part3_col=3 +PREHOOK: Input: default@part_table4 +PREHOOK: Input: default@part_table4@part4_col=1 +PREHOOK: Input: default@part_table4@part4_col=2 +PREHOOK: Input: default@part_table4@part4_col=3 +PREHOOK: Input: default@part_table4@part4_col=4 +PREHOOK: Input: default@part_table5 +PREHOOK: Input: default@part_table5@part5_col=1 +PREHOOK: Input: default@part_table5@part5_col=2 +PREHOOK: Input: default@part_table5@part5_col=3 +PREHOOK: Input: default@part_table5@part5_col=4 +PREHOOK: Input: default@part_table5@part5_col=5 +PREHOOK: Input: default@reg_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM part_table1 pt1, + part_table2 pt2, + part_table3 pt3, + part_table4 pt4, + part_table5 pt5, + reg_table rt +WHERE rt.col = pt1.part1_col +AND pt2.part2_col = pt1.part1_col +AND pt3.part3_col = pt1.part1_col +AND pt4.part4_col = pt1.part1_col +AND pt5.part5_col = pt1.part1_col +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_table1 +POSTHOOK: Input: default@part_table1@part1_col=1 +POSTHOOK: Input: default@part_table2 +POSTHOOK: Input: default@part_table2@part2_col=1 +POSTHOOK: Input: default@part_table2@part2_col=2 +POSTHOOK: Input: default@part_table3 +POSTHOOK: Input: default@part_table3@part3_col=1 +POSTHOOK: Input: default@part_table3@part3_col=2 +POSTHOOK: Input: default@part_table3@part3_col=3 +POSTHOOK: Input: default@part_table4 +POSTHOOK: Input: default@part_table4@part4_col=1 +POSTHOOK: Input: default@part_table4@part4_col=2 +POSTHOOK: Input: default@part_table4@part4_col=3 +POSTHOOK: Input: default@part_table4@part4_col=4 +POSTHOOK: Input: default@part_table5 +POSTHOOK: Input: default@part_table5@part5_col=1 +POSTHOOK: Input: default@part_table5@part5_col=2 +POSTHOOK: Input: default@part_table5@part5_col=3 +POSTHOOK: Input: default@part_table5@part5_col=4 +POSTHOOK: Input: default@part_table5@part5_col=5 +POSTHOOK: Input: default@reg_table +#### A masked pattern was here #### +1 1 1 1 1 1 1 1 1 1 1 +PREHOOK: query: DROP TABLE part_table1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_table1 +PREHOOK: Output: default@part_table1 +POSTHOOK: query: DROP TABLE part_table1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_table1 +POSTHOOK: Output: default@part_table1 +PREHOOK: query: DROP TABLE part_table2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_table2 +PREHOOK: Output: default@part_table2 +POSTHOOK: query: DROP TABLE part_table2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_table2 +POSTHOOK: Output: default@part_table2 +PREHOOK: query: DROP TABLE part_table3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_table3 +PREHOOK: Output: default@part_table3 +POSTHOOK: query: DROP TABLE part_table3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_table3 +POSTHOOK: Output: default@part_table3 +PREHOOK: query: DROP TABLE part_table4 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_table4 +PREHOOK: Output: default@part_table4 +POSTHOOK: query: DROP TABLE part_table4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_table4 +POSTHOOK: Output: default@part_table4 +PREHOOK: query: DROP TABLE part_table5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_table5 +PREHOOK: Output: default@part_table5 +POSTHOOK: query: DROP TABLE part_table5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_table5 +POSTHOOK: Output: default@part_table5 +PREHOOK: query: DROP TABLE reg_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@reg_table +PREHOOK: Output: default@reg_table +POSTHOOK: query: DROP TABLE reg_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@reg_table +POSTHOOK: Output: default@reg_table