diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java index ac7e8e35f9..3076f6f611 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory; -import org.apache.hadoop.hive.metastore.StatObjectConverter; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; @@ -102,22 +101,25 @@ public ColumnStatisticsObj aggregate(List colStatsWit (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats(); lowerBound = Math.max(lowerBound, newData.getNumDVs()); higherBound += newData.getNumDVs(); - densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils - .decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); + if ((newData.getHighValue() != null) && (newData.getLowValue() != null)) { + densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils + .decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); + } if (ndvEstimator != null) { ndvEstimator.mergeEstimators(newData.getNdvEstimator()); } if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils - .decimalToDouble(newData.getLowValue())) { + if ((aggregateData.getLowValue() != null) && (newData.getLowValue() != null) && (MetaStoreUtils + .decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils.decimalToDouble(newData.getLowValue()))) { aggregateData.setLowValue(aggregateData.getLowValue()); } else { aggregateData.setLowValue(newData.getLowValue()); } - if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils - .decimalToDouble(newData.getHighValue())) { + if ((aggregateData.getHighValue() != null) && (newData.getHighValue() != null) + && (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils + .decimalToDouble(newData.getHighValue()))) { aggregateData.setHighValue(aggregateData.getHighValue()); } else { aggregateData.setHighValue(newData.getHighValue()); @@ -132,13 +134,15 @@ public ColumnStatisticsObj aggregate(List colStatsWit // to get a good estimation. aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues()); } else { - long estimation; + long estimation = 0; if (useDensityFunctionForNDVEstimation) { // We have estimation, lowerbound and higherbound. We use estimation // if it is between lowerbound and higherbound. double densityAvg = densityAvgSum / partNames.size(); - estimation = (long) ((MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils - .decimalToDouble(aggregateData.getLowValue())) / densityAvg); + if ((aggregateData.getHighValue() != null) && (aggregateData.getLowValue() != null)) { + estimation = (long) ((MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) + - MetaStoreUtils.decimalToDouble(aggregateData.getLowValue())) / densityAvg); + } if (estimation < lowerBound) { estimation = lowerBound; } else if (estimation > higherBound) { @@ -170,8 +174,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit String partName = csp.getPartName(); DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats(); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils - .decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); + if ((newData.getHighValue() != null) && (newData.getLowValue() != null)) { + densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) + - MetaStoreUtils.decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); + } } adjustedIndexMap.put(partName, (double) indexMap.get(partName)); adjustedStatsMap.put(partName, cso.getStatsData()); @@ -201,8 +207,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit csd.setDecimalStats(aggregateData); adjustedStatsMap.put(pseudoPartName.toString(), csd); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils - .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); + if ((aggregateData.getHighValue() != null) && (aggregateData.getLowValue() != null)) { + densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) + - MetaStoreUtils.decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); + } } // reset everything pseudoPartName = new StringBuilder(); @@ -220,14 +228,16 @@ public ColumnStatisticsObj aggregate(List colStatsWit if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils - .decimalToDouble(newData.getLowValue())) { + if ((aggregateData.getLowValue() != null) && (newData.getLowValue() != null) + && (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils + .decimalToDouble(newData.getLowValue()))) { aggregateData.setLowValue(aggregateData.getLowValue()); } else { aggregateData.setLowValue(newData.getLowValue()); } - if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils - .decimalToDouble(newData.getHighValue())) { + if ((aggregateData.getHighValue() != null) && (newData.getHighValue() != null) + && (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils + .decimalToDouble(newData.getHighValue()))) { aggregateData.setHighValue(aggregateData.getHighValue()); } else { aggregateData.setHighValue(newData.getHighValue()); @@ -244,8 +254,10 @@ public ColumnStatisticsObj aggregate(List colStatsWit csd.setDecimalStats(aggregateData); adjustedStatsMap.put(pseudoPartName.toString(), csd); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils - .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); + if ((aggregateData.getHighValue() != null) && (aggregateData.getLowValue() != null)) { + densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) + - MetaStoreUtils.decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); + } } } }