diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5fcc367cc9..89a2576ba8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7351,7 +7351,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) dpCtx = qbm.getDPCtx(dest); if (dpCtx == null) { destinationTable.validatePartColumnNames(partSpec, false); - dpCtx = new DynamicPartitionCtx(partSpec, + dpCtx = new DynamicPartitionCtx(partSpec, destinationTable.getPartitionKeys(), conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME), conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE)); qbm.setDPCtx(dest, dpCtx); @@ -7675,6 +7675,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) cols = ct.cols; colTypes = ct.colTypes; dpCtx = new DynamicPartitionCtx(partitionColumnNames, + destinationTable != null ? destinationTable.getPartitionKeys() : null, conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME), conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE)); qbm.setDPCtx(dest, dpCtx); @@ -8313,7 +8314,7 @@ private DynamicPartitionCtx checkDynPart(QB qb, QBMetaData qbm, Table dest_tab, DynamicPartitionCtx dpCtx = qbm.getDPCtx(dest); if (dpCtx == null) { dest_tab.validatePartColumnNames(partSpec, false); - dpCtx = new DynamicPartitionCtx(partSpec, + dpCtx = new DynamicPartitionCtx(partSpec, parts, conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME), conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE)); qbm.setDPCtx(dest, dpCtx); @@ -8456,18 +8457,50 @@ private Operator genConversionSelectOperator(String dest, QB qb, Operator input, } expressions.add(column); } - } - // deal with dynamic partition columns: convert ExprNodeDesc type to String?? - if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) { - // DP columns starts with tableFields.size() - for (int i = tableFields.size() + (updating(dest) ? 1 : 0); i < rowFields.size(); ++i) { - TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); - ExprNodeDesc column = new ExprNodeColumnDesc( - rowFieldTypeInfo, rowFields.get(i).getInternalName(), "", true); - expressions.add(column); + // deal with dynamic partition columns + if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) { + // rowFields contains non-partitioned columns (tableFields) followed by DP columns + int rowFieldsOffset = tableFields.size() + (updating(dest) ? 1 : 0); + for (int dpColIdx = 0; dpColIdx < rowFields.size() - rowFieldsOffset; ++dpColIdx) { + + // create ExprNodeDesc + ColumnInfo inputColumn = rowFields.get(dpColIdx + rowFieldsOffset); + TypeInfo inputTypeInfo = inputColumn.getType(); + ExprNodeDesc column = + new ExprNodeColumnDesc(inputTypeInfo, inputColumn.getInternalName(), "", true); + + // Cast input column to destination column type if necessary. + if (dpCtx.getDpFieldSchemas() != null && dpCtx.getDpFieldSchemas().size() > 0) { + String destPartitionName = dpCtx.getDPColNames().get(dpColIdx); + FieldSchema destPartitionFieldSchema = dpCtx.getDpFieldSchemas().get(destPartitionName); + if (destPartitionFieldSchema != null) { + String partitionType = destPartitionFieldSchema.getType(); + if (partitionType != null) { + PrimitiveTypeInfo partitionTypeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(partitionType); + + if (!partitionTypeInfo.equals(inputTypeInfo)) { + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, partitionTypeInfo); + converted = true; + } + } else { + LOG.warn( + "Couldn't get FieldSchema for partition" + destPartitionFieldSchema.getName()); + } + } else { + LOG.warn("Partition schema for dynamic partition " + destPartitionName + + " not found in DynamicPartitionCtx."); + } + } else { + LOG.info("Partition schema for dynamic partition " + inputColumn.getAlias() + " (" + + inputColumn.getInternalName() + + ") not found in DynamicPartitionCtx. This is expected with a CTAS."); + } + expressions.add(column); + } } - // converted = true; // [TODO]: should we check & convert type to String and set it to true? } if (converted) { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java index c1aeb8f136..d50aab7331 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java @@ -19,6 +19,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -39,6 +41,7 @@ private static final long serialVersionUID = 1L; private Map partSpec; // partSpec is an ORDERED hash map + private Map dpFieldSchemas; // name and FieldSchema of dynamic partitions private int numDPCols; // number of dynamic partition columns private int numSPCols; // number of static partition columns private String spPath; // path name corresponding to SP columns @@ -59,8 +62,8 @@ public DynamicPartitionCtx() { * partitioned columns, which will all be dynamic partitions since the binding * is done after executing the query in the CTAS. */ - public DynamicPartitionCtx(List partColNames, String defaultPartName, - int maxParts) throws SemanticException { + public DynamicPartitionCtx(List partColNames, List dpFieldSchemas, + String defaultPartName, int maxParts) throws SemanticException { this.partSpec = new LinkedHashMap<>(); this.spNames = new ArrayList<>(); this.dpNames = new ArrayList<>(); @@ -82,10 +85,12 @@ public DynamicPartitionCtx(List partColNames, String defaultPartName, throw new SemanticException(e); } this.whiteListPattern = confVal == null || confVal.isEmpty() ? null : Pattern.compile(confVal); + populateDpFieldSchemas(dpFieldSchemas); } - public DynamicPartitionCtx(Map partSpec, String defaultPartName, - int maxParts) throws SemanticException { + + public DynamicPartitionCtx(Map partSpec, List dpFieldSchemas, + String defaultPartName, int maxParts) throws SemanticException { this.partSpec = partSpec; this.spNames = new ArrayList(); this.dpNames = new ArrayList(); @@ -114,6 +119,7 @@ public DynamicPartitionCtx(Map partSpec, String defaultPartName, throw new SemanticException(e); } this.whiteListPattern = confVal == null || confVal.isEmpty() ? null : Pattern.compile(confVal); + populateDpFieldSchemas(dpFieldSchemas); } public DynamicPartitionCtx(DynamicPartitionCtx dp) { @@ -213,4 +219,17 @@ public void setSPPath(String sp) { public String getSPPath() { return this.spPath; } + + private void populateDpFieldSchemas(List dynamicPartitionColumns) { + dpFieldSchemas = new HashMap<>(); + if (dynamicPartitionColumns != null) { + dynamicPartitionColumns.forEach(dynamicPartitionColumn -> { + dpFieldSchemas.put(dynamicPartitionColumn.getName(), dynamicPartitionColumn); + }); + } + } + + public Map getDpFieldSchemas() { + return dpFieldSchemas; + } } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java index 2c4b69b2fe..897fb2b6e0 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java @@ -283,7 +283,8 @@ private FileSinkOperator getFileSink(AcidUtils.Operation writeType, partCols.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, PARTCOL_NAME, "a", true)); Map partColMap= new LinkedHashMap(1); partColMap.put(PARTCOL_NAME, null); - DynamicPartitionCtx dpCtx = new DynamicPartitionCtx(partColMap, "Sunday", 100); + DynamicPartitionCtx dpCtx = new DynamicPartitionCtx(partColMap, + null, "Sunday", 100); //todo: does this need the finalDestination? desc = new FileSinkDesc(basePath, tableDesc, false, 1, false, false, 1, 1, partCols, dpCtx, null, null, false, false, false, false); diff --git ql/src/test/queries/clientpositive/dynpart_cast.q ql/src/test/queries/clientpositive/dynpart_cast.q new file mode 100644 index 0000000000..0ec2220845 --- /dev/null +++ ql/src/test/queries/clientpositive/dynpart_cast.q @@ -0,0 +1,9 @@ +set hive.stats.autogather=true; + +drop table dynpart_cast; +create table dynpart_cast (i int) PARTITIONED BY (`static_part` int, `dyn_part` int); + +-- stats task will fail here if dynamic partition not cast to integer and creates "dyn_part=002" +INSERT INTO TABLE dynpart_cast PARTITION (static_part=03, dyn_part) +SELECT 1, +'002'; diff --git ql/src/test/results/clientpositive/autoColumnStats_6.q.out ql/src/test/results/clientpositive/autoColumnStats_6.q.out index da3be3e5bb..ff708cb6b0 100644 --- ql/src/test/results/clientpositive/autoColumnStats_6.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_6.q.out @@ -41,11 +41,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: key, value, one, two, three Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -92,21 +92,21 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: string) null sort order: aa sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col3 (type: int) - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/dynpart_cast.q.out ql/src/test/results/clientpositive/dynpart_cast.q.out new file mode 100644 index 0000000000..8d4b706db6 --- /dev/null +++ ql/src/test/results/clientpositive/dynpart_cast.q.out @@ -0,0 +1,25 @@ +PREHOOK: query: drop table dynpart_cast +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table dynpart_cast +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table dynpart_cast (i int) PARTITIONED BY (`static_part` int, `dyn_part` int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dynpart_cast +POSTHOOK: query: create table dynpart_cast (i int) PARTITIONED BY (`static_part` int, `dyn_part` int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dynpart_cast +PREHOOK: query: INSERT INTO TABLE dynpart_cast PARTITION (static_part=03, dyn_part) +SELECT 1, +'002' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dynpart_cast@static_part=3 +POSTHOOK: query: INSERT INTO TABLE dynpart_cast PARTITION (static_part=03, dyn_part) +SELECT 1, +'002' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dynpart_cast@static_part=3/dyn_part=2 +POSTHOOK: Lineage: dynpart_cast PARTITION(static_part=3,dyn_part=2).i SIMPLE [] diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 43bb789840..0f09b4972d 100644 --- ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -31,8 +31,10 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -46,26 +48,65 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string) - null sort order: aaaa - sort order: ++++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + key expressions: _col1 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), UDFToInteger(VALUE._col3) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 732000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2000 Data size: 732000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int) + outputColumnNames: key, value, ds, hr + Statistics: Num rows: 2000 Data size: 732000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string), hr (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 4272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Move Operator @@ -80,7 +121,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -88,3 +129,34 @@ STAGE PLANS: Column Types: string, string Table: default.non_acid + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 4 Data size: 4272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 4272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 4272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 4272 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + diff --git ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out index f745b46899..52fd083565 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), CAST( VALUE._col2 AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) outputColumnNames: key, value, hr Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -128,21 +128,21 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out index fc9050b2c3..8625e56ec4 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out @@ -228,20 +228,20 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 236 Data size: 43392 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs @@ -255,14 +255,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -380,14 +380,14 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 236 Data size: 43392 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -404,14 +404,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -639,14 +639,14 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 236 Data size: 43392 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 225 Data size: 41369 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -663,14 +663,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 247 Data size: 45505 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 95aae7286f..0e4ac5e7e7 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -1172,20 +1172,20 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo filterExpr: ((key = 'foo') and (ds = '2008-04-08') and (hr = 11)) (type: boolean) - Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1601 Data size: 139287 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 435 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1194,10 +1194,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1280,20 +1280,20 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo filterExpr: ((key = 'foo') and (ds = '2008-04-08') and (hr >= 11)) (type: boolean) - Statistics: Num rows: 3201 Data size: 313458 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3201 Data size: 291291 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 455 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), hr (type: int) outputColumnNames: _col0, _col4 - Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: '2008-04-08' (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) null sort order: aaaa sort order: ++++ Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL value expressions: 'foo' (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1306,7 +1306,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 3480 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1426,20 +1426,20 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo filterExpr: (value = 'bar') (type: boolean) - Statistics: Num rows: 4200 Data size: 1247197 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4200 Data size: 1171800 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (value = 'bar') (type: boolean) - Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 3906 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 3696 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int), _bucket_number (type: string), _col0 (type: struct) null sort order: aaaa sort order: ++++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 3696 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1451,7 +1451,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 3696 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1631,20 +1631,20 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo_no_cp filterExpr: ((key = 'foo') and (ds = '2008-04-08') and (hr = 11)) (type: boolean) - Statistics: Num rows: 1601 Data size: 599036 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1601 Data size: 440275 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), key (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2190 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) null sort order: aaaa sort order: ++++ Map-reduce partition columns: _col3 (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2190 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1657,7 +1657,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1740,20 +1740,20 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo_no_cp filterExpr: ((key = 'foo') and (ds = '2008-04-08') and (hr >= 11)) (type: boolean) - Statistics: Num rows: 3201 Data size: 1197516 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3201 Data size: 880275 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 2750 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), key (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 4380 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) null sort order: aaaa sort order: ++++ Map-reduce partition columns: _col3 (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 4380 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1766,7 +1766,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 4380 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/llap/llap_smb.q.out ql/src/test/results/clientpositive/llap/llap_smb.q.out index 24026d0bab..d456aa6c60 100644 --- ql/src/test/results/clientpositive/llap/llap_smb.q.out +++ ql/src/test/results/clientpositive/llap/llap_smb.q.out @@ -269,10 +269,10 @@ STAGE PLANS: TableScan alias: a filterExpr: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Merge Join Operator condition map: Inner Join 0 to 1 @@ -280,20 +280,20 @@ STAGE PLANS: 0 id (type: bigint) 1 id (type: bigint) outputColumnNames: _col2, _col3 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col2 (type: int), _col3 (type: smallint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: smallint) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: smallint) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Execution mode: llap Reducer 2 @@ -304,10 +304,10 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/llap/orc_merge1.q.out ql/src/test/results/clientpositive/llap/orc_merge1.q.out index 9da73e65ac..44a5d6f290 100644 --- ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -73,19 +73,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -211,19 +211,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1b_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -389,19 +389,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1c_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index a6ea33493f..3bf3710c69 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -74,11 +74,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -96,11 +96,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -128,12 +128,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -222,11 +222,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -244,11 +244,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -276,12 +276,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -415,11 +415,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -437,11 +437,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -469,12 +469,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/llap/orc_merge2.q.out ql/src/test/results/clientpositive/llap/orc_merge2.q.out index 9b0d3b4234..19ca90c2ad 100644 --- ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -48,11 +48,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: key, value, one, two, three Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -70,11 +70,11 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: struct), _col4 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: string) null sort order: aa sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col3 (type: int) - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -102,12 +102,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index d35f44b10a..3215f09ac8 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -74,11 +74,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -96,11 +96,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -128,12 +128,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -222,11 +222,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -244,11 +244,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -276,12 +276,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -415,11 +415,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -437,11 +437,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -469,12 +469,12 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out index fcff20a68e..59f4acd59b 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out @@ -48,11 +48,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: key, value, one, two, three Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -70,11 +70,11 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: struct), _col4 (type: struct) Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: string) null sort order: aa sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col3 (type: int) - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -102,12 +102,12 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat diff --git ql/src/test/results/clientpositive/llap/tez_dml.q.out ql/src/test/results/clientpositive/llap/tez_dml.q.out index 4ad78d8582..d716b63012 100644 --- ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -489,19 +489,19 @@ STAGE PLANS: alias: tmp_src Statistics: Num rows: 309 Data size: 30591 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: value (type: string), cnt (type: bigint) + expressions: value (type: string), UDFToInteger(cnt) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 309 Data size: 30591 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 309 Data size: 30591 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmp_src_part Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: c, d Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator diff --git ql/src/test/results/clientpositive/orc_merge1.q.out ql/src/test/results/clientpositive/orc_merge1.q.out index 9c07816340..8a4aade0d4 100644 --- ql/src/test/results/clientpositive/orc_merge1.q.out +++ ql/src/test/results/clientpositive/orc_merge1.q.out @@ -66,19 +66,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -185,19 +185,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1b_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -343,19 +343,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orcfile_merge1c_n1 Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator diff --git ql/src/test/results/clientpositive/orc_merge10.q.out ql/src/test/results/clientpositive/orc_merge10.q.out index 4a5f03c82f..0b9b6649c5 100644 --- ql/src/test/results/clientpositive/orc_merge10.q.out +++ ql/src/test/results/clientpositive/orc_merge10.q.out @@ -67,11 +67,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -118,20 +118,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -210,11 +210,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -261,20 +261,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -392,11 +392,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -443,20 +443,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/orc_merge2.q.out ql/src/test/results/clientpositive/orc_merge2.q.out index d132d62b18..2997a4f272 100644 --- ql/src/test/results/clientpositive/orc_merge2.q.out +++ ql/src/test/results/clientpositive/orc_merge2.q.out @@ -41,11 +41,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: key, value, one, two, three Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -92,20 +92,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: string) null sort order: aa sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col3 (type: int) - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out index 7f9a04b09f..3806c36390 100644 --- ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out @@ -67,11 +67,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -118,20 +118,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -210,11 +210,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -261,20 +261,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -392,11 +392,11 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string) outputColumnNames: key, value, ds, part Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator @@ -443,20 +443,20 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/smb_join_partition_key.q.out ql/src/test/results/clientpositive/smb_join_partition_key.q.out index c18d01d26a..34246d1803 100644 --- ql/src/test/results/clientpositive/smb_join_partition_key.q.out +++ ql/src/test/results/clientpositive/smb_join_partition_key.q.out @@ -109,20 +109,20 @@ PREHOOK: Output: default@smb_table_part POSTHOOK: query: INSERT OVERWRITE TABLE smb_table_part PARTITION (p1) SELECT key, value, 100 as p1 FROM data_table POSTHOOK: type: QUERY POSTHOOK: Input: default@data_table -POSTHOOK: Output: default@smb_table_part@p1=100 -POSTHOOK: Lineage: smb_table_part PARTITION(p1=100).key SIMPLE [(data_table)data_table.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: smb_table_part PARTITION(p1=100).value SIMPLE [(data_table)data_table.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Output: default@smb_table_part@p1=100.0 +POSTHOOK: Lineage: smb_table_part PARTITION(p1=100.0).key SIMPLE [(data_table)data_table.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_table_part PARTITION(p1=100.0).value SIMPLE [(data_table)data_table.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: SELECT s1.key, s2.p1 FROM smb_table s1 INNER JOIN smb_table_part s2 ON s1.key = s2.key ORDER BY s1.key PREHOOK: type: QUERY PREHOOK: Input: default@smb_table PREHOOK: Input: default@smb_table_part -PREHOOK: Input: default@smb_table_part@p1=100 +PREHOOK: Input: default@smb_table_part@p1=100.0 #### A masked pattern was here #### POSTHOOK: query: SELECT s1.key, s2.p1 FROM smb_table s1 INNER JOIN smb_table_part s2 ON s1.key = s2.key ORDER BY s1.key POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_table POSTHOOK: Input: default@smb_table_part -POSTHOOK: Input: default@smb_table_part@p1=100 +POSTHOOK: Input: default@smb_table_part@p1=100.0 #### A masked pattern was here #### 1 100.0 2 100.0 diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out index bc6c3add54..82de9a463d 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out @@ -451,12 +451,20 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 -169 val_169 NULL day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 -374 val_374 NULL day1 1 +374 val_374 val_374 day1 1 +374 val_374 val_374 day1 1 PREHOOK: query: drop table bucket_big_n17 PREHOOK: type: DROPTABLE PREHOOK: Input: default@bucket_big_n17 diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark index 67b62c1265..6ef49d92d0 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark @@ -451,12 +451,20 @@ POSTHOOK: Input: default@bucket_small_n17@pri=2 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 -169 val_169 NULL day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 +169 val_169 val_169 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 -374 val_374 NULL day1 1 +374 val_374 val_374 day1 1 +374 val_374 val_374 day1 1 PREHOOK: query: drop table bucket_big_n17 PREHOOK: type: DROPTABLE PREHOOK: Input: default@bucket_big_n17 diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out index 56d5ed945b..9efcf98dd8 100644 --- ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out @@ -63,21 +63,21 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), CAST( VALUE._col2 AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/spark/orc_merge1.q.out ql/src/test/results/clientpositive/spark/orc_merge1.q.out index 977c4cbfc1..5c95429418 100644 --- ql/src/test/results/clientpositive/spark/orc_merge1.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge1.q.out @@ -69,7 +69,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -152,7 +152,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -279,7 +279,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/orc_merge2.q.out ql/src/test/results/clientpositive/spark/orc_merge2.q.out index 4647b86ea3..089be29c53 100644 --- ql/src/test/results/clientpositive/spark/orc_merge2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge2.q.out @@ -50,20 +50,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: string) null sort order: aa sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out index b7d3dd725d..d58642db70 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out @@ -71,20 +71,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false @@ -168,20 +168,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false @@ -310,20 +310,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 2) AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col2 (type: string) null sort order: a sort order: + - Map-reduce partition columns: _col2 (type: int) + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false