commit c98190376c83ecaa49064a785f6379e8e123ab0e Author: Owen O'Malley Date: Fri May 22 11:38:11 2015 -0700 HIVE-10796. Move the complex types out of JavaDataModel. diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java index 274330b..8482e18 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java @@ -435,7 +435,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc public int estimate() { JavaDataModel model = JavaDataModel.get(); return model.lengthFor(columnType) + model.primitive1() + model.primitive2() + - model.lengthFor(numDV); + numDV.lengthFor(model); } protected void initNDVEstimator(int numBitVectors) { @@ -833,7 +833,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc public int estimate() { JavaDataModel model = JavaDataModel.get(); return model.primitive1() * 2 + model.primitive2() * 4 + - model.lengthFor(columnType) + model.lengthFor(numDV); + model.lengthFor(columnType) + numDV.lengthFor(model); } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java index 008d937..434956f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java @@ -238,7 +238,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep NumericHistogram histogram; // the histogram object @Override public int estimate() { - return JavaDataModel.get().lengthFor(histogram); + return histogram.lengthFor(JavaDataModel.get()); } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java index e53b893..89d95f8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java @@ -358,7 +358,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep @Override public int estimate() { JavaDataModel model = JavaDataModel.get(); - return model.lengthFor(histogram) + + return histogram.lengthFor(model) + model.array() + JavaDataModel.PRIMITIVES2 * quantiles.length; } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java index 8212bea..7cc5734 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.io.Text; public class NumDistinctValueEstimator { @@ -356,4 +357,21 @@ public long estimateNumDistinctValues() { numDistinctValues = Math.pow(2.0, avgLeastSigZero); return ((long)(numDistinctValues)); } + + public int lengthFor(JavaDataModel model) { + int length = model.object(); + length += model.primitive1() * 2; // two int + length += model.primitive2(); // one double + length += model.lengthForRandom() * 2; // two Random + + int numVector = getnumBitVectors(); + if (numVector > 0) { + length += model.array() * 3; // three array + length += model.primitive1() * numVector * 2; // two int array + length += (model.object() + model.array() + model.primitive1() + + model.primitive2()) * numVector; // bitset array + } + return length; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java index 0577a4f..4182234 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.Random; +import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; @@ -311,4 +312,17 @@ public double quantile(double q) { public int getNumBins() { return bins == null ? 0 : bins.size(); } + + public int lengthFor(JavaDataModel model) { + int length = model.object(); + length += model.primitive1() * 2; // two int + int numBins = getNumBins(); + if (numBins > 0) { + length += model.arrayList(); // List + // Coord holds two doubles + length += numBins * (model.object() + model.primitive2() * 2); + } + length += model.lengthForRandom(); // Random + return length; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java index 3352a08..a0cd188 100644 --- ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java +++ ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.util; -import org.apache.hadoop.hive.ql.udf.generic.NumDistinctValueEstimator; -import org.apache.hadoop.hive.ql.udf.generic.NumericHistogram; - /** * Estimation of memory footprint of object */ @@ -160,33 +157,6 @@ public int lengthFor(String string) { return lengthForStringOfLength(string.length()); } - public int lengthFor(NumericHistogram histogram) { - int length = object(); - length += primitive1() * 2; // two int - int numBins = histogram.getNumBins(); - if (numBins > 0) { - length += arrayList(); // List - length += numBins * (object() + primitive2() * 2); // Coord holds two doubles - } - length += lengthForRandom(); // Random - return length; - } - - public int lengthFor(NumDistinctValueEstimator estimator) { - int length = object(); - length += primitive1() * 2; // two int - length += primitive2(); // one double - length += lengthForRandom() * 2; // two Random - - int numVector = estimator.getnumBitVectors(); - if (numVector > 0) { - length += array() * 3; // three array - length += primitive1() * numVector * 2; // two int array - length += (object() + array() + primitive1() + primitive2()) * numVector; // bitset array - } - return length; - } - public int lengthForRandom() { // boolean + double + AtomicLong return object() + primitive1() + primitive2() + object() + primitive2();