diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java index cc18dabf4d..1797b650da 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils; import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.merge.DecimalColumnStatsMerger; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; @@ -110,18 +111,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - if (MetaStoreServerUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreServerUtils - .decimalToDouble(newData.getLowValue())) { - aggregateData.setLowValue(aggregateData.getLowValue()); - } else { - aggregateData.setLowValue(newData.getLowValue()); - } - if (MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreServerUtils - .decimalToDouble(newData.getHighValue())) { - aggregateData.setHighValue(aggregateData.getHighValue()); - } else { - aggregateData.setHighValue(newData.getHighValue()); - } + DecimalColumnStatsMerger merger = new DecimalColumnStatsMerger(); + merger.setLowValue(aggregateData, newData); + merger.setHighValue(aggregateData, newData); + aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java index 7c217fc50f..2fce86fbbc 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.merge.DoubleColumnStatsMerger; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,9 +108,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData - .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); + DoubleColumnStatsMerger merger = new DoubleColumnStatsMerger(); + merger.setLowValue(aggregateData, newData); + merger.setHighValue(aggregateData, newData); + aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java index 2b237e5816..ffde02455a 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.merge.LongColumnStatsMerger; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,9 +107,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData - .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); + LongColumnStatsMerger merger = new LongColumnStatsMerger(); + merger.setLowValue(aggregateData, newData); + merger.setHighValue(aggregateData, newData); + aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java index a114188110..5cb62bd636 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java @@ -60,8 +60,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new aggregateColStats.getStatsData().setDecimalStats(aggregateData); } - @VisibleForTesting - void setLowValue(DecimalColumnStatsDataInspector aggregateData, DecimalColumnStatsDataInspector newData) { + public void setLowValue(DecimalColumnStatsDataInspector aggregateData, DecimalColumnStatsDataInspector newData) { if (!aggregateData.isSetLowValue() && !newData.isSetLowValue()) { return; } @@ -79,8 +78,7 @@ void setLowValue(DecimalColumnStatsDataInspector aggregateData, DecimalColumnSta aggregateData.setLowValue(mergedLowValue); } - @VisibleForTesting - void setHighValue(DecimalColumnStatsDataInspector aggregateData, DecimalColumnStatsDataInspector newData) { + public void setHighValue(DecimalColumnStatsDataInspector aggregateData, DecimalColumnStatsDataInspector newData) { if (!aggregateData.isSetHighValue() && !newData.isSetHighValue()) { return; } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java index a02f25b709..41f9095645 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java @@ -54,7 +54,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new aggregateColStats.getStatsData().setDoubleStats(aggregateData); } - private void setLowValue(DoubleColumnStatsDataInspector aggregateData, DoubleColumnStatsDataInspector newData) { + public void setLowValue(DoubleColumnStatsDataInspector aggregateData, DoubleColumnStatsDataInspector newData) { if (!aggregateData.isSetLowValue() && !newData.isSetLowValue()) { return; } @@ -64,13 +64,13 @@ private void setLowValue(DoubleColumnStatsDataInspector aggregateData, DoubleCol aggregateData.setLowValue(lowValue); } - private void setHighValue(DoubleColumnStatsDataInspector aggregateData, DoubleColumnStatsDataInspector newData) { + public void setHighValue(DoubleColumnStatsDataInspector aggregateData, DoubleColumnStatsDataInspector newData) { if (!aggregateData.isSetHighValue() && !newData.isSetHighValue()) { return; } double highValue = Math.max( aggregateData.isSetHighValue() ? aggregateData.getHighValue() : Double.MIN_VALUE, newData.isSetHighValue() ? newData.getHighValue() : Double.MIN_VALUE); - aggregateData.setLowValue(highValue); + aggregateData.setHighValue(highValue); } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java index 67adbf1e98..dfc8421058 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java @@ -54,7 +54,7 @@ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj new aggregateColStats.getStatsData().setLongStats(aggregateData); } - private void setLowValue(LongColumnStatsDataInspector aggregateData, LongColumnStatsDataInspector newData) { + public void setLowValue(LongColumnStatsDataInspector aggregateData, LongColumnStatsDataInspector newData) { if (!aggregateData.isSetLowValue() && !newData.isSetLowValue()) { return; } @@ -64,7 +64,7 @@ private void setLowValue(LongColumnStatsDataInspector aggregateData, LongColumnS aggregateData.setLowValue(lowValue); } - private void setHighValue(LongColumnStatsDataInspector aggregateData, LongColumnStatsDataInspector newData) { + public void setHighValue(LongColumnStatsDataInspector aggregateData, LongColumnStatsDataInspector newData) { if (!aggregateData.isSetHighValue() && !newData.isSetHighValue()) { return; }