diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java index 60b71aa..1510fdd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java @@ -18,12 +18,6 @@ package org.apache.hadoop.hive.ql.exec.tez; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; - -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; - import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -171,20 +165,24 @@ private void initialize() throws SerDeException { List tables = work.getEventSourceTableDescMap().get(s); // Real column name - on which the operation is being performed List columnNames = work.getEventSourceColumnNameMap().get(s); + // Column type + List columnTypes = work.getEventSourceColumnTypeMap().get(s); // Expression for the operation. e.g. N^2 > 10 List partKeyExprs = work.getEventSourcePartKeyExprMap().get(s); // eventSourceTableDesc, eventSourceColumnName, evenSourcePartKeyExpr move in lock-step. // One entry is added to each at the same time Iterator cit = columnNames.iterator(); + Iterator typit = columnTypes.iterator(); Iterator pit = partKeyExprs.iterator(); // A single source can process multiple columns, and will send an event for each of them. for (TableDesc t : tables) { numExpectedEventsPerSource.get(s).decrement(); ++sourceInfoCount; String columnName = cit.next(); + String columnType = typit.next(); ExprNodeDesc partKeyExpr = pit.next(); - SourceInfo si = createSourceInfo(t, partKeyExpr, columnName, jobConf); + SourceInfo si = createSourceInfo(t, partKeyExpr, columnName, columnType, jobConf); if (!sourceInfoMap.containsKey(s)) { sourceInfoMap.put(s, new ArrayList()); } @@ -248,37 +246,25 @@ protected void prunePartitionSingleSource(String source, SourceInfo si) LOG.debug(sb.toString()); } - ObjectInspector targetOi = findTargetOi(si.partKey, si.columnName); - Converter converter = ObjectInspectorConverters.getConverter( - PrimitiveObjectInspectorFactory.javaStringObjectInspector, targetOi); + ObjectInspector oi = + PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(TypeInfoFactory + .getPrimitiveTypeInfo(si.columnType)); - StructObjectInspector soi = ObjectInspectorFactory.getStandardStructObjectInspector( - Collections.singletonList(columnName), Collections.singletonList(targetOi)); + Converter converter = + ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi); + + StructObjectInspector soi = + ObjectInspectorFactory.getStandardStructObjectInspector( + Collections.singletonList(columnName), Collections.singletonList(oi)); @SuppressWarnings("rawtypes") ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(si.partKey); - eval.initialize(soi); // We expect the row with just the relevant column. + eval.initialize(soi); applyFilterToPartitions(converter, eval, columnName, values); } - private ObjectInspector findTargetOi(ExprNodeDesc expr, String columnName) { - if (expr instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc colExpr = (ExprNodeColumnDesc)expr; - // TODO: this is not necessarily going to work for all cases. At least, table name is needed. - // Also it's not clear if this is going to work with subquery columns and such. - if (columnName.equals(colExpr.getColumn())) { - return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - (PrimitiveTypeInfo)colExpr.getTypeInfo()); - } - } - for (ExprNodeDesc child : expr.getChildren()) { - ObjectInspector oi = findTargetOi(child, columnName); - if (oi != null) return oi; - } - return null; - } - @SuppressWarnings("rawtypes") private void applyFilterToPartitions(Converter converter, ExprNodeEvaluator eval, String columnName, Set values) throws HiveException { @@ -322,9 +308,9 @@ private void applyFilterToPartitions(Converter converter, ExprNodeEvaluator eval @VisibleForTesting protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, - JobConf jobConf) throws + String columnType, JobConf jobConf) throws SerDeException { - return new SourceInfo(t, partKeyExpr, columnName, jobConf); + return new SourceInfo(t, partKeyExpr, columnName, columnType, jobConf); } @@ -341,18 +327,20 @@ protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, Str /* Whether to skipPruning - depends on the payload from an event which may signal skip - if the event payload is too large */ public AtomicBoolean skipPruning = new AtomicBoolean(); public final String columnName; + public final String columnType; @VisibleForTesting // Only used for testing. - SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobConf jobConf, Object forTesting) { + SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf, Object forTesting) { this.partKey = partKey; this.columnName = columnName; + this.columnType = columnType; this.deserializer = null; this.soi = null; this.field = null; this.fieldInspector = null; } - public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobConf jobConf) + public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.skipPruning.set(false); @@ -360,6 +348,7 @@ public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobC this.partKey = partKey; this.columnName = columnName; + this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index d2a5948..f151871 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -484,13 +484,17 @@ public boolean equals(Object obj) { return partKeys; } - public boolean isPartitionKey(String colName) { + public FieldSchema getPartColByName(String colName) { for (FieldSchema key : getPartCols()) { if (key.getName().toLowerCase().equals(colName)) { - return true; + return key; } } - return false; + return null; + } + + public boolean isPartitionKey(String colName) { + return getPartColByName(colName) == null ? false : true; } // TODO merge this with getBucketCols function diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index 292d375..8d232ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -200,6 +200,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje Table table = ts.getConf().getTableMetadata(); if (table != null && table.isPartitionKey(column)) { + String columnType = table.getPartColByName(column).getType(); String alias = ts.getConf().getAlias(); PrunedPartitionList plist = parseContext.getPrunedPartitions(alias, ts); if (LOG.isDebugEnabled()) { @@ -213,7 +214,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje } if (plist == null || plist.getPartitions().size() != 0) { LOG.info("Dynamic partitioning: " + table.getCompleteName() + "." + column); - generateEventOperatorPlan(ctx, parseContext, ts, column); + generateEventOperatorPlan(ctx, parseContext, ts, column, columnType); } else { // all partitions have been statically removed LOG.debug("No partition pruning necessary."); @@ -269,7 +270,7 @@ private void cleanTableScanFilters(TableScanOperator ts) throws SemanticExceptio } private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext parseContext, - TableScanOperator ts, String column) { + TableScanOperator ts, String column, String columnType) { // we will put a fork in the plan at the source of the reduce sink Operator parentOfRS = ctx.generator.getParentOperators().get(0); @@ -329,6 +330,7 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars eventDesc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils .getFieldSchemasFromColumnList(keyExprs, "key"))); eventDesc.setTargetColumnName(column); + eventDesc.setTargetColumnType(columnType); eventDesc.setPartKey(partKey); OperatorFactory.getAndMakeChild(eventDesc, groupByOp); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 70912e0..8eab3af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -421,6 +421,12 @@ public static void processAppMasterEvent( List columns = work.getEventSourceColumnNameMap().get(sourceName); columns.add(eventDesc.getTargetColumnName()); + if (!work.getEventSourceColumnTypeMap().containsKey(sourceName)) { + work.getEventSourceColumnTypeMap().put(sourceName, new LinkedList()); + } + List columnTypes = work.getEventSourceColumnTypeMap().get(sourceName); + columnTypes.add(eventDesc.getTargetColumnType()); + // store partition key expr in map-work if (!work.getEventSourcePartKeyExprMap().containsKey(sourceName)) { work.getEventSourcePartKeyExprMap().put(sourceName, new LinkedList()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java index 59a012e..73bbebd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java @@ -32,6 +32,9 @@ // column in the target table that will be pruned against private String targetColumnName; + // type of target column + private String targetColumnType; + // tableScan is only available during compile private transient TableScanOperator tableScan; @@ -47,6 +50,10 @@ public void setTableScan(TableScanOperator tableScan) { } @Explain(displayName = "Target column") + public String displayTargetColumn() { + return targetColumnName + " (" + targetColumnType + ")"; + } + public String getTargetColumnName() { return targetColumnName; } @@ -55,6 +62,14 @@ public void setTargetColumnName(String columnName) { this.targetColumnName = columnName; } + public String getTargetColumnType() { + return targetColumnType; + } + + public void setTargetColumnType(String columnType) { + this.targetColumnType = columnType; + } + @Override public void writeEventHeader(DataOutputBuffer buffer) throws IOException { super.writeEventHeader(buffer); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index edafdb1..97c28f4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -123,6 +123,8 @@ new LinkedHashMap>(); private Map> eventSourceColumnNameMap = new LinkedHashMap>(); + private Map> eventSourceColumnTypeMap = + new LinkedHashMap>(); private Map> eventSourcePartKeyExprMap = new LinkedHashMap>(); @@ -568,6 +570,10 @@ public void setEventSourceColumnNameMap(Map> map) { return eventSourceColumnNameMap; } + public Map> getEventSourceColumnTypeMap() { + return eventSourceColumnTypeMap; + } + public Map> getEventSourcePartKeyExprMap() { return eventSourcePartKeyExprMap; } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java index 1ce2e09..a46f10d 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java @@ -510,10 +510,10 @@ public DynamicPartitionPrunerForEventTesting( } @Override - protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, + protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, String columnType, JobConf jobConf) throws SerDeException { - return new SourceInfo(t, partKeyExpr, columnName, jobConf, null); + return new SourceInfo(t, partKeyExpr, columnName, columnType, jobConf, null); } @Override diff --git ql/src/test/queries/clientpositive/dynamic_partition_pruning.q ql/src/test/queries/clientpositive/dynamic_partition_pruning.q index 6009c74..0c2ba83 100644 --- ql/src/test/queries/clientpositive/dynamic_partition_pruning.q +++ ql/src/test/queries/clientpositive/dynamic_partition_pruning.q @@ -25,6 +25,14 @@ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) set hive.tez.dynamic.partition.pruning=true; select count(*) from srcpart where ds = '2008-04-08'; +-- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +set hive.tez.dynamic.partition.pruning=false; +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +set hive.tez.dynamic.partition.pruning=true; + -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11; @@ -121,6 +129,10 @@ EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'; select count(*) from srcpart where ds = '2008-04-08'; +-- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; + -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11; diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 79348f3..02d4fdd 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -1459,10 +1459,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: b Partition key expr: ds Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 2 Execution mode: llap Map 2 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 17d527b..a673c28 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -248,10 +248,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -438,6 +438,251 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 +PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds is not null and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -503,10 +748,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 6 @@ -537,10 +782,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -849,10 +1094,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -864,10 +1109,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1114,10 +1359,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1322,7 +1567,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr is not null and UDFToDouble(hr) is not null) (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: UDFToDouble(hr) is not null (type: boolean) @@ -1365,10 +1610,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1494,10 +1739,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1864,10 +2109,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2056,10 +2301,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2277,10 +2522,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2292,10 +2537,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2420,10 +2665,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2511,17 +2756,16 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: llap Map 4 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[5]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2596,7 +2840,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[6]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2636,10 +2879,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2708,7 +2951,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[12]) (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2748,10 +2990,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 6 @@ -3130,10 +3372,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: uber @@ -3166,10 +3408,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3336,10 +3578,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: uber @@ -3372,10 +3614,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3526,10 +3768,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3541,10 +3783,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: uber @@ -3621,10 +3863,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3636,10 +3878,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3748,10 +3990,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -3808,6 +4050,132 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + Map 3 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Execution mode: llap + Reducer 2 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -3896,10 +4264,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 4 @@ -3930,10 +4298,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4067,10 +4435,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4082,10 +4450,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4216,10 +4584,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4272,7 +4640,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr is not null and UDFToDouble(hr) is not null) (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: UDFToDouble(hr) is not null (type: boolean) @@ -4329,10 +4697,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4453,10 +4821,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4723,10 +5091,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4804,7 +5172,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[5]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4860,7 +5227,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[6]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4951,7 +5317,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[12]) (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -5015,10 +5380,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 4 @@ -5342,10 +5707,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: uber @@ -5378,10 +5743,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5524,10 +5889,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5539,10 +5904,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 0bd466e..49a1efc 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -227,10 +227,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -765,10 +765,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -917,10 +917,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Map 4 @@ -980,10 +980,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 4 Execution mode: llap Union 2 diff --git ql/src/test/results/clientpositive/llap/metadataonly1.q.out ql/src/test/results/clientpositive/llap/metadataonly1.q.out index 40f122a..e1dd7f2 100644 --- ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -926,10 +926,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 5fbc4f6..2bec8d6 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -248,10 +248,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -503,10 +503,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 6 @@ -537,10 +537,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -849,10 +849,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -864,10 +864,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1114,10 +1114,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1322,7 +1322,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr is not null and UDFToDouble(hr) is not null) (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: UDFToDouble(hr) is not null (type: boolean) @@ -1365,10 +1365,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1494,10 +1494,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1864,10 +1864,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2056,10 +2056,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2277,10 +2277,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2292,10 +2292,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2420,10 +2420,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2511,17 +2511,16 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: vectorized, llap Map 4 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[5]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2596,7 +2595,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[6]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2636,10 +2634,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2708,7 +2706,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[12]) (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -2748,10 +2745,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 6 @@ -3130,10 +3127,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized, uber @@ -3166,10 +3163,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3336,10 +3333,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized, uber @@ -3372,10 +3369,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3526,10 +3523,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3541,10 +3538,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: vectorized, uber @@ -3621,10 +3618,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3636,10 +3633,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3748,10 +3745,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -3896,10 +3893,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 4 @@ -3930,10 +3927,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4067,10 +4064,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4082,10 +4079,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4216,10 +4213,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4272,7 +4269,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr is not null and UDFToDouble(hr) is not null) (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: UDFToDouble(hr) is not null (type: boolean) @@ -4329,10 +4326,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4453,10 +4450,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4723,10 +4720,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4804,7 +4801,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[5]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4860,7 +4856,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[6]) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4951,7 +4946,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds) IN (RS[12]) (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -5015,10 +5009,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 4 @@ -5342,10 +5336,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: vectorized, uber @@ -5378,10 +5372,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5524,10 +5518,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5539,10 +5533,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index 9582334..36f4eb7 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -1423,10 +1423,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: b Partition key expr: ds Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 2 Map 2 Map Operator Tree: diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index d4b670d..483aec0 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -245,10 +245,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -428,6 +428,243 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 +PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds is not null and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -492,10 +729,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 6 Map Operator Tree: @@ -525,10 +762,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -826,10 +1063,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -841,10 +1078,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1083,10 +1320,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1326,10 +1563,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1451,10 +1688,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1809,10 +2046,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1993,10 +2230,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2209,10 +2446,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2224,10 +2461,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2348,10 +2585,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2436,10 +2673,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Map 4 Map Operator Tree: @@ -2554,10 +2791,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2661,10 +2898,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 6 Map Operator Tree: @@ -3026,10 +3263,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Reduce Operator Tree: @@ -3061,10 +3298,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3225,10 +3462,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Reduce Operator Tree: @@ -3260,10 +3497,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3409,10 +3646,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3424,10 +3661,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Reduce Operator Tree: @@ -3500,10 +3737,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3515,10 +3752,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3626,10 +3863,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -3684,6 +3921,129 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: day(ds) is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Map 3 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((date = '2008-04-08') and ds is not null) and day(ds) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -3771,10 +4131,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -3804,10 +4164,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -3938,10 +4298,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -3953,10 +4313,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4084,10 +4444,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4194,10 +4554,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4315,10 +4675,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4578,10 +4938,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4858,10 +5218,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -5173,10 +5533,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Reduce Operator Tree: @@ -5208,10 +5568,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5353,10 +5713,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5368,10 +5728,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index bc5638c..0d8e2bd 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -226,10 +226,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -752,10 +752,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -900,10 +900,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -961,10 +961,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 4 Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/tez/metadataonly1.q.out ql/src/test/results/clientpositive/tez/metadataonly1.q.out index 4ef71f8..37408e1 100644 --- ql/src/test/results/clientpositive/tez/metadataonly1.q.out +++ ql/src/test/results/clientpositive/tez/metadataonly1.q.out @@ -920,10 +920,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 diff --git ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index 4f6cb91..20fdee7 100644 --- ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -246,10 +246,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -497,10 +497,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 6 @@ -531,10 +531,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -837,10 +837,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -852,10 +852,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1098,10 +1098,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1345,10 +1345,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1472,10 +1472,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1836,10 +1836,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2024,10 +2024,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2242,10 +2242,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2257,10 +2257,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2383,10 +2383,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2473,10 +2473,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: vectorized Map 4 @@ -2593,10 +2593,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2702,10 +2702,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 6 @@ -3076,10 +3076,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized @@ -3112,10 +3112,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3278,10 +3278,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized @@ -3314,10 +3314,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3464,10 +3464,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3479,10 +3479,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: vectorized @@ -3558,10 +3558,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3573,10 +3573,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3684,10 +3684,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -3831,10 +3831,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 4 @@ -3865,10 +3865,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4001,10 +4001,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4016,10 +4016,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4149,10 +4149,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4261,10 +4261,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4384,10 +4384,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4651,10 +4651,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4937,10 +4937,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 4 @@ -5261,10 +5261,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -5297,10 +5297,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5442,10 +5442,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5457,10 +5457,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: