diff --git ql/src/test/results/clientpositive/alter_table_update_status.q.out ql/src/test/results/clientpositive/alter_table_update_status.q.out index 6453391466..e643863fe4 100644 --- ql/src/test/results/clientpositive/alter_table_update_status.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status.q.out @@ -339,7 +339,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name s data_type smallint -min 0 +min 3 max 3 num_nulls 1 distinct_count 1 @@ -358,7 +358,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name i data_type int -min 0 +min 45 max 45 num_nulls 1 distinct_count 1 @@ -377,7 +377,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name b data_type bigint -min 0 +min 456 max 456 num_nulls 1 distinct_count 1 @@ -396,7 +396,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name f data_type float -min 0.0 +min 45454.3984375 max 45454.3984375 num_nulls 1 distinct_count 1 @@ -415,7 +415,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name d data_type double -min 0.0 +min 454.6565 max 454.6565 num_nulls 1 distinct_count 1 @@ -453,7 +453,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name ts data_type timestamp -min 0 +min 1325379723 max 1325379723 num_nulls 1 distinct_count 1 @@ -586,7 +586,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name t data_type tinyint -min 0 +min 2 max 2 num_nulls 1 distinct_count 1 @@ -632,7 +632,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name s data_type smallint -min 0 +min 3 max 3 num_nulls 1 distinct_count 1 @@ -678,7 +678,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name i data_type int -min 0 +min 45 max 45 num_nulls 1 distinct_count 1 @@ -724,7 +724,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name b data_type bigint -min 0 +min 456 max 456 num_nulls 1 distinct_count 1 @@ -770,7 +770,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name f data_type float -min 0.0 +min 45454.3984375 max 45454.3984375 num_nulls 1 distinct_count 1 @@ -816,7 +816,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name d data_type double -min 0.0 +min 454.6565 max 454.6565 num_nulls 1 distinct_count 1 @@ -908,7 +908,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats_n0 col_name ts data_type timestamp -min 0 +min 1325379723 max 1325379723 num_nulls 1 distinct_count 1 diff --git ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out index 068f30220b..904aa1f00c 100644 --- ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out @@ -339,7 +339,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name s data_type smallint -min 0 +min 3 max 3 num_nulls 1 distinct_count 1 @@ -358,7 +358,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name i data_type int -min 0 +min 45 max 45 num_nulls 1 distinct_count 1 @@ -377,7 +377,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name b data_type bigint -min 0 +min 456 max 456 num_nulls 1 distinct_count 1 @@ -396,7 +396,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name f data_type float -min 0.0 +min 45454.3984375 max 45454.3984375 num_nulls 1 distinct_count 1 @@ -415,7 +415,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name d data_type double -min 0.0 +min 454.6565 max 454.6565 num_nulls 1 distinct_count 1 @@ -453,7 +453,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name ts data_type timestamp -min 0 +min 1325379723 max 1325379723 num_nulls 1 distinct_count 1 @@ -586,7 +586,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name t data_type tinyint -min 0 +min 2 max 2 num_nulls 1 distinct_count 1 @@ -632,7 +632,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name s data_type smallint -min 0 +min 3 max 3 num_nulls 1 distinct_count 1 @@ -678,7 +678,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name i data_type int -min 0 +min 45 max 45 num_nulls 1 distinct_count 1 @@ -724,7 +724,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name b data_type bigint -min 0 +min 456 max 456 num_nulls 1 distinct_count 1 @@ -770,7 +770,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name f data_type float -min 0.0 +min 45454.3984375 max 45454.3984375 num_nulls 1 distinct_count 1 @@ -816,7 +816,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name d data_type double -min 0.0 +min 454.6565 max 454.6565 num_nulls 1 distinct_count 1 @@ -908,7 +908,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@datatype_stats col_name ts data_type timestamp -min 0 +min 1325379723 max 1325379723 num_nulls 1 distinct_count 1 diff --git ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out index e3dabce817..a6ccdd7d78 100644 --- ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out +++ ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -117,7 +117,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 input vertices: 1 Map 2 - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -125,13 +125,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 2] - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/vector_coalesce_3.q.out ql/src/test/results/clientpositive/vector_coalesce_3.q.out index 0890943733..1438ef8735 100644 --- ql/src/test/results/clientpositive/vector_coalesce_3.q.out +++ ql/src/test/results/clientpositive/vector_coalesce_3.q.out @@ -131,7 +131,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col2 - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -139,13 +139,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1] - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java index bcdb56d1e2..a69d7fc5d4 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java @@ -54,6 +54,8 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); aggregateData.setNumDVs(ndv); } + + aggregateColStats.getStatsData().setDateStats(aggregateData); } private Date min(Date v1, Date v2) { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java index 50943588e3..9b87f20b7d 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java @@ -29,10 +29,8 @@ public class DecimalColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DecimalColumnStatsDataInspector aggregateData = - decimalInspectorFromStats(aggregateColStats); - DecimalColumnStatsDataInspector newData = - decimalInspectorFromStats(newColStats); + DecimalColumnStatsDataInspector aggregateData = decimalInspectorFromStats(aggregateColStats); + DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(newColStats); Decimal lowValue = getMin(aggregateData.getLowValue(), newData.getLowValue()); aggregateData.setLowValue(lowValue); @@ -59,6 +57,8 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); aggregateData.setNumDVs(ndv); } + + aggregateColStats.getStatsData().setDecimalStats(aggregateData); } Decimal getMax(Decimal firstValue, Decimal secondValue) { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java index cbacacd626..dd6cd1c3fe 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java @@ -30,7 +30,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { DoubleColumnStatsDataInspector aggregateData = doubleInspectorFromStats(aggregateColStats); DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(newColStats); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); + setMinValue(aggregateData, newData); aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { @@ -50,5 +50,17 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); aggregateData.setNumDVs(ndv); } + + aggregateColStats.getStatsData().setDoubleStats(aggregateData); + } + + private void setMinValue(DoubleColumnStatsDataInspector aggregateData, DoubleColumnStatsDataInspector newData) { + if (!aggregateData.isSetLowValue() && !newData.isSetLowValue()) { + return; + } + double lowValue = Math.min( + aggregateData.isSetLowValue() ? aggregateData.getLowValue() : Double.MAX_VALUE, + newData.isSetLowValue() ? newData.getLowValue() : Double.MAX_VALUE); + aggregateData.setLowValue(lowValue); } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java index 8e70371e69..e8706245ac 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java @@ -30,7 +30,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { LongColumnStatsDataInspector aggregateData = longInspectorFromStats(aggregateColStats); LongColumnStatsDataInspector newData = longInspectorFromStats(newColStats); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); + setMinValue(aggregateData, newData); aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { @@ -50,5 +50,17 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); aggregateData.setNumDVs(ndv); } + + aggregateColStats.getStatsData().setLongStats(aggregateData); + } + + private void setMinValue(LongColumnStatsDataInspector aggregateData, LongColumnStatsDataInspector newData) { + if (!aggregateData.isSetLowValue() && !newData.isSetLowValue()) { + return; + } + long lowValue = Math.min( + aggregateData.isSetLowValue() ? aggregateData.getLowValue() : Long.MAX_VALUE, + newData.isSetLowValue() ? newData.getLowValue() : Long.MAX_VALUE); + aggregateData.setLowValue(lowValue); } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java index 762685d00d..dec44852f2 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java @@ -50,5 +50,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); aggregateData.setNumDVs(ndv); } + + aggregateColStats.getStatsData().setStringStats(aggregateData); } }