diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java index 60b71aa..1510fdd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java @@ -18,12 +18,6 @@ package org.apache.hadoop.hive.ql.exec.tez; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; - -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; - import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -171,20 +165,24 @@ private void initialize() throws SerDeException { List tables = work.getEventSourceTableDescMap().get(s); // Real column name - on which the operation is being performed List columnNames = work.getEventSourceColumnNameMap().get(s); + // Column type + List columnTypes = work.getEventSourceColumnTypeMap().get(s); // Expression for the operation. e.g. N^2 > 10 List partKeyExprs = work.getEventSourcePartKeyExprMap().get(s); // eventSourceTableDesc, eventSourceColumnName, evenSourcePartKeyExpr move in lock-step. // One entry is added to each at the same time Iterator cit = columnNames.iterator(); + Iterator typit = columnTypes.iterator(); Iterator pit = partKeyExprs.iterator(); // A single source can process multiple columns, and will send an event for each of them. for (TableDesc t : tables) { numExpectedEventsPerSource.get(s).decrement(); ++sourceInfoCount; String columnName = cit.next(); + String columnType = typit.next(); ExprNodeDesc partKeyExpr = pit.next(); - SourceInfo si = createSourceInfo(t, partKeyExpr, columnName, jobConf); + SourceInfo si = createSourceInfo(t, partKeyExpr, columnName, columnType, jobConf); if (!sourceInfoMap.containsKey(s)) { sourceInfoMap.put(s, new ArrayList()); } @@ -248,37 +246,25 @@ protected void prunePartitionSingleSource(String source, SourceInfo si) LOG.debug(sb.toString()); } - ObjectInspector targetOi = findTargetOi(si.partKey, si.columnName); - Converter converter = ObjectInspectorConverters.getConverter( - PrimitiveObjectInspectorFactory.javaStringObjectInspector, targetOi); + ObjectInspector oi = + PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(TypeInfoFactory + .getPrimitiveTypeInfo(si.columnType)); - StructObjectInspector soi = ObjectInspectorFactory.getStandardStructObjectInspector( - Collections.singletonList(columnName), Collections.singletonList(targetOi)); + Converter converter = + ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi); + + StructObjectInspector soi = + ObjectInspectorFactory.getStandardStructObjectInspector( + Collections.singletonList(columnName), Collections.singletonList(oi)); @SuppressWarnings("rawtypes") ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(si.partKey); - eval.initialize(soi); // We expect the row with just the relevant column. + eval.initialize(soi); applyFilterToPartitions(converter, eval, columnName, values); } - private ObjectInspector findTargetOi(ExprNodeDesc expr, String columnName) { - if (expr instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc colExpr = (ExprNodeColumnDesc)expr; - // TODO: this is not necessarily going to work for all cases. At least, table name is needed. - // Also it's not clear if this is going to work with subquery columns and such. - if (columnName.equals(colExpr.getColumn())) { - return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - (PrimitiveTypeInfo)colExpr.getTypeInfo()); - } - } - for (ExprNodeDesc child : expr.getChildren()) { - ObjectInspector oi = findTargetOi(child, columnName); - if (oi != null) return oi; - } - return null; - } - @SuppressWarnings("rawtypes") private void applyFilterToPartitions(Converter converter, ExprNodeEvaluator eval, String columnName, Set values) throws HiveException { @@ -322,9 +308,9 @@ private void applyFilterToPartitions(Converter converter, ExprNodeEvaluator eval @VisibleForTesting protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, - JobConf jobConf) throws + String columnType, JobConf jobConf) throws SerDeException { - return new SourceInfo(t, partKeyExpr, columnName, jobConf); + return new SourceInfo(t, partKeyExpr, columnName, columnType, jobConf); } @@ -341,18 +327,20 @@ protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, Str /* Whether to skipPruning - depends on the payload from an event which may signal skip - if the event payload is too large */ public AtomicBoolean skipPruning = new AtomicBoolean(); public final String columnName; + public final String columnType; @VisibleForTesting // Only used for testing. - SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobConf jobConf, Object forTesting) { + SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf, Object forTesting) { this.partKey = partKey; this.columnName = columnName; + this.columnType = columnType; this.deserializer = null; this.soi = null; this.field = null; this.fieldInspector = null; } - public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobConf jobConf) + public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.skipPruning.set(false); @@ -360,6 +348,7 @@ public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobC this.partKey = partKey; this.columnName = columnName; + this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index d2a5948..f151871 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -484,13 +484,17 @@ public boolean equals(Object obj) { return partKeys; } - public boolean isPartitionKey(String colName) { + public FieldSchema getPartColByName(String colName) { for (FieldSchema key : getPartCols()) { if (key.getName().toLowerCase().equals(colName)) { - return true; + return key; } } - return false; + return null; + } + + public boolean isPartitionKey(String colName) { + return getPartColByName(colName) == null ? false : true; } // TODO merge this with getBucketCols function diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index 292d375..8d232ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -200,6 +200,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje Table table = ts.getConf().getTableMetadata(); if (table != null && table.isPartitionKey(column)) { + String columnType = table.getPartColByName(column).getType(); String alias = ts.getConf().getAlias(); PrunedPartitionList plist = parseContext.getPrunedPartitions(alias, ts); if (LOG.isDebugEnabled()) { @@ -213,7 +214,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje } if (plist == null || plist.getPartitions().size() != 0) { LOG.info("Dynamic partitioning: " + table.getCompleteName() + "." + column); - generateEventOperatorPlan(ctx, parseContext, ts, column); + generateEventOperatorPlan(ctx, parseContext, ts, column, columnType); } else { // all partitions have been statically removed LOG.debug("No partition pruning necessary."); @@ -269,7 +270,7 @@ private void cleanTableScanFilters(TableScanOperator ts) throws SemanticExceptio } private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext parseContext, - TableScanOperator ts, String column) { + TableScanOperator ts, String column, String columnType) { // we will put a fork in the plan at the source of the reduce sink Operator parentOfRS = ctx.generator.getParentOperators().get(0); @@ -329,6 +330,7 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars eventDesc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils .getFieldSchemasFromColumnList(keyExprs, "key"))); eventDesc.setTargetColumnName(column); + eventDesc.setTargetColumnType(columnType); eventDesc.setPartKey(partKey); OperatorFactory.getAndMakeChild(eventDesc, groupByOp); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 70912e0..8eab3af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -421,6 +421,12 @@ public static void processAppMasterEvent( List columns = work.getEventSourceColumnNameMap().get(sourceName); columns.add(eventDesc.getTargetColumnName()); + if (!work.getEventSourceColumnTypeMap().containsKey(sourceName)) { + work.getEventSourceColumnTypeMap().put(sourceName, new LinkedList()); + } + List columnTypes = work.getEventSourceColumnTypeMap().get(sourceName); + columnTypes.add(eventDesc.getTargetColumnType()); + // store partition key expr in map-work if (!work.getEventSourcePartKeyExprMap().containsKey(sourceName)) { work.getEventSourcePartKeyExprMap().put(sourceName, new LinkedList()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java index 59a012e..73bbebd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java @@ -32,6 +32,9 @@ // column in the target table that will be pruned against private String targetColumnName; + // type of target column + private String targetColumnType; + // tableScan is only available during compile private transient TableScanOperator tableScan; @@ -47,6 +50,10 @@ public void setTableScan(TableScanOperator tableScan) { } @Explain(displayName = "Target column") + public String displayTargetColumn() { + return targetColumnName + " (" + targetColumnType + ")"; + } + public String getTargetColumnName() { return targetColumnName; } @@ -55,6 +62,14 @@ public void setTargetColumnName(String columnName) { this.targetColumnName = columnName; } + public String getTargetColumnType() { + return targetColumnType; + } + + public void setTargetColumnType(String columnType) { + this.targetColumnType = columnType; + } + @Override public void writeEventHeader(DataOutputBuffer buffer) throws IOException { super.writeEventHeader(buffer); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index e1abf37..ffaf129 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -124,6 +124,8 @@ new LinkedHashMap>(); private Map> eventSourceColumnNameMap = new LinkedHashMap>(); + private Map> eventSourceColumnTypeMap = + new LinkedHashMap>(); private Map> eventSourcePartKeyExprMap = new LinkedHashMap>(); @@ -584,6 +586,10 @@ public void setEventSourceColumnNameMap(Map> map) { return eventSourceColumnNameMap; } + public Map> getEventSourceColumnTypeMap() { + return eventSourceColumnTypeMap; + } + public Map> getEventSourcePartKeyExprMap() { return eventSourcePartKeyExprMap; } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java index 1ce2e09..080ee11 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestDynamicPartitionPruner.java @@ -454,6 +454,7 @@ private MapWork createMockMapWork(TestSource... testSources) { Map> tableMap = new HashMap<>(); Map> columnMap = new HashMap<>(); + Map> typeMap = new HashMap<>(); Map> exprMap = new HashMap<>(); int count = 0; @@ -474,6 +475,13 @@ private MapWork createMockMapWork(TestSource... testSources) { } columnList.add(testSource.vertexName + "c_" + count + "_" + i); + List typeList = typeMap.get(testSource.vertexName); + if (typeList == null) { + typeList = new LinkedList<>(); + typeMap.put(testSource.vertexName, typeList); + } + typeList.add("string"); + List exprNodeDescList = exprMap.get(testSource.vertexName); if (exprNodeDescList == null) { exprNodeDescList = new LinkedList<>(); @@ -488,6 +496,7 @@ private MapWork createMockMapWork(TestSource... testSources) { doReturn(tableMap).when(mapWork).getEventSourceTableDescMap(); doReturn(columnMap).when(mapWork).getEventSourceColumnNameMap(); doReturn(exprMap).when(mapWork).getEventSourcePartKeyExprMap(); + doReturn(typeMap).when(mapWork).getEventSourceColumnTypeMap(); return mapWork; } @@ -510,10 +519,10 @@ public DynamicPartitionPrunerForEventTesting( } @Override - protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, + protected SourceInfo createSourceInfo(TableDesc t, ExprNodeDesc partKeyExpr, String columnName, String columnType, JobConf jobConf) throws SerDeException { - return new SourceInfo(t, partKeyExpr, columnName, jobConf, null); + return new SourceInfo(t, partKeyExpr, columnName, columnType, jobConf, null); } @Override diff --git ql/src/test/queries/clientpositive/dynamic_partition_pruning.q ql/src/test/queries/clientpositive/dynamic_partition_pruning.q index 6009c74..0c2ba83 100644 --- ql/src/test/queries/clientpositive/dynamic_partition_pruning.q +++ ql/src/test/queries/clientpositive/dynamic_partition_pruning.q @@ -25,6 +25,14 @@ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) set hive.tez.dynamic.partition.pruning=true; select count(*) from srcpart where ds = '2008-04-08'; +-- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +set hive.tez.dynamic.partition.pruning=false; +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +set hive.tez.dynamic.partition.pruning=true; + -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11; @@ -121,6 +129,10 @@ EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'; select count(*) from srcpart where ds = '2008-04-08'; +-- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; +select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'; + -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11; diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 79348f3..02d4fdd 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -1459,10 +1459,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: b Partition key expr: ds Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 2 Execution mode: llap Map 2 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index a7e974a..d2c11d7 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -248,10 +248,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -438,6 +438,245 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 +PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -503,10 +742,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 6 @@ -537,10 +776,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -849,10 +1088,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -864,10 +1103,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1114,10 +1353,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1362,10 +1601,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1488,10 +1727,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -1849,10 +2088,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2041,10 +2280,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2262,10 +2501,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2277,10 +2516,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2405,10 +2644,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2496,10 +2735,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: llap Map 4 @@ -2619,10 +2858,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -2730,10 +2969,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 6 @@ -3112,10 +3351,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: uber @@ -3148,10 +3387,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3318,10 +3557,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: uber @@ -3354,10 +3593,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3508,10 +3747,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3523,10 +3762,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: uber @@ -3603,10 +3842,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3618,10 +3857,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3730,10 +3969,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -3790,6 +4029,129 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + Map 3 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Execution mode: llap + Reducer 2 + Execution mode: uber + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -3878,10 +4240,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 4 @@ -3912,10 +4274,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4049,10 +4411,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4064,10 +4426,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4198,10 +4560,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4308,10 +4670,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4429,10 +4791,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4699,10 +5061,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -4988,10 +5350,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: llap Map 4 @@ -5315,10 +5677,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: uber @@ -5351,10 +5713,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5493,10 +5855,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5508,10 +5870,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 0bd466e..49a1efc 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -227,10 +227,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -765,10 +765,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Reducer 2 @@ -917,10 +917,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Execution mode: llap Map 4 @@ -980,10 +980,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 4 Execution mode: llap Union 2 diff --git ql/src/test/results/clientpositive/llap/metadataonly1.q.out ql/src/test/results/clientpositive/llap/metadataonly1.q.out index 40f122a..e1dd7f2 100644 --- ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -926,10 +926,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 2464076..63121a9 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -248,10 +248,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -503,10 +503,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 6 @@ -537,10 +537,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -849,10 +849,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -864,10 +864,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1114,10 +1114,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1362,10 +1362,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1488,10 +1488,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -1849,10 +1849,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2041,10 +2041,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2262,10 +2262,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2277,10 +2277,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2405,10 +2405,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2496,10 +2496,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: vectorized, llap Map 4 @@ -2619,10 +2619,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -2730,10 +2730,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 6 @@ -3112,10 +3112,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized, uber @@ -3148,10 +3148,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3318,10 +3318,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized, uber @@ -3354,10 +3354,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3508,10 +3508,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3523,10 +3523,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: vectorized, uber @@ -3603,10 +3603,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3618,10 +3618,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3730,10 +3730,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -3878,10 +3878,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 4 @@ -3912,10 +3912,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4049,10 +4049,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4064,10 +4064,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4198,10 +4198,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4308,10 +4308,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4429,10 +4429,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4699,10 +4699,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Reducer 2 @@ -4988,10 +4988,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized, llap Map 4 @@ -5315,10 +5315,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: vectorized, uber @@ -5351,10 +5351,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5493,10 +5493,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5508,10 +5508,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: llap Reducer 2 diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index 9582334..36f4eb7 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -1423,10 +1423,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: b Partition key expr: ds Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 2 Map 2 Map Operator Tree: diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index caa4398..58ab2c0 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -245,10 +245,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -428,6 +428,237 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 +PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -492,10 +723,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 6 Map Operator Tree: @@ -525,10 +756,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -826,10 +1057,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -841,10 +1072,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1083,10 +1314,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1323,10 +1554,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1445,10 +1676,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1794,10 +2025,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -1978,10 +2209,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2194,10 +2425,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2209,10 +2440,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2333,10 +2564,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2421,10 +2652,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Map 4 Map Operator Tree: @@ -2539,10 +2770,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -2646,10 +2877,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 6 Map Operator Tree: @@ -3011,10 +3242,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Reduce Operator Tree: @@ -3046,10 +3277,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3210,10 +3441,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Reduce Operator Tree: @@ -3245,10 +3476,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3394,10 +3625,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3409,10 +3640,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Reduce Operator Tree: @@ -3485,10 +3716,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3500,10 +3731,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3611,10 +3842,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -3669,6 +3900,126 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 +PREHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: -- single column, single key, udf with typechange +EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Map 3 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: day(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: day(ds) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Map 1 + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Input: default@srcpart_date +#### A masked pattern was here #### +1000 PREHOOK: query: -- multiple sources, single key EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 @@ -3756,10 +4107,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -3789,10 +4140,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -3923,10 +4274,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -3938,10 +4289,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4069,10 +4420,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4176,10 +4527,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4294,10 +4645,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4557,10 +4908,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -4837,10 +5188,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -5152,10 +5503,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Reduce Operator Tree: @@ -5187,10 +5538,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5328,10 +5679,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5343,10 +5694,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index bc5638c..0d8e2bd 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -226,10 +226,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -752,10 +752,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: @@ -900,10 +900,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 1 Map 4 Map Operator Tree: @@ -961,10 +961,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: dim_shops_id (int) Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target column: dim_shops_id Target Vertex: Map 4 Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/tez/metadataonly1.q.out ql/src/test/results/clientpositive/tez/metadataonly1.q.out index 4ef71f8..37408e1 100644 --- ql/src/test/results/clientpositive/tez/metadataonly1.q.out +++ ql/src/test/results/clientpositive/tez/metadataonly1.q.out @@ -920,10 +920,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 diff --git ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index 3cb52be..cda76db 100644 --- ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -246,10 +246,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -497,10 +497,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 6 @@ -531,10 +531,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -837,10 +837,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -852,10 +852,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1098,10 +1098,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1342,10 +1342,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1466,10 +1466,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -1821,10 +1821,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2009,10 +2009,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Stage: Stage-0 @@ -2227,10 +2227,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -2242,10 +2242,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2368,10 +2368,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2458,10 +2458,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 4 Execution mode: vectorized Map 4 @@ -2578,10 +2578,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -2687,10 +2687,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 6 @@ -3061,10 +3061,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized @@ -3097,10 +3097,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3263,10 +3263,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 8 Execution mode: vectorized @@ -3299,10 +3299,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -3449,10 +3449,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3464,10 +3464,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Reducer 2 Execution mode: vectorized @@ -3543,10 +3543,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col0 (type: string) @@ -3558,10 +3558,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 5 Union 3 Vertex: Union 3 @@ -3669,10 +3669,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -3816,10 +3816,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 4 @@ -3850,10 +3850,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -3986,10 +3986,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: _col2 (type: string) @@ -4001,10 +4001,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4134,10 +4134,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4243,10 +4243,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4363,10 +4363,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (string) Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4630,10 +4630,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Reducer 2 @@ -4916,10 +4916,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Execution mode: vectorized Map 4 @@ -5240,10 +5240,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -5276,10 +5276,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart Partition key expr: ds Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -5417,10 +5417,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: ds (string) Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: ds Target Vertex: Map 1 Select Operator expressions: UDFToDouble(_col2) (type: double) @@ -5432,10 +5432,10 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator + Target column: hr (int) Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: NONE - Target column: hr Target Vertex: Map 1 Reducer 2 Reduce Operator Tree: