commit 02efc92a1af8b0fcc4c01510f983a2c566b42a34 Author: kellyzly Date: Thu Jul 6 23:11:44 2017 -0400 HIVE-17010.3.patch diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java index 4924df7..e808a4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java @@ -22,7 +22,6 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.TerminalOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.ObjectPair; @@ -34,6 +33,7 @@ import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.TerminalOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession; @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.stats.StatsUtils; /** * SetSparkReducerParallelism determines how many reducers should @@ -126,7 +127,7 @@ public Object process(Node nd, Stack stack, for (Operator sibling : sink.getChildOperators().get(0).getParentOperators()) { if (sibling.getStatistics() != null) { - numberOfBytes += sibling.getStatistics().getDataSize(); + numberOfBytes = StatsUtils.safeAdd(numberOfBytes, sibling.getStatistics().getDataSize()); if (LOG.isDebugEnabled()) { LOG.debug("Sibling " + sibling + " has stats: " + sibling.getStatistics()); } @@ -143,7 +144,7 @@ public Object process(Node nd, Stack stack, OperatorUtils.findOperatorsUpstream(sibling, TableScanOperator.class); for (TableScanOperator source : sources) { if (source.getStatistics() != null) { - numberOfBytes += source.getStatistics().getDataSize(); + numberOfBytes = StatsUtils.safeAdd(numberOfBytes, source.getStatistics().getDataSize()); if (LOG.isDebugEnabled()) { LOG.debug("Table source " + source + " has stats: " + source.getStatistics()); }