commit fc6d74cda22b2e52a2b64a7c21288cf994822c4e Author: Sahil Takiar Date: Fri Jan 5 11:01:41 2018 -0800 HIVE-18378: Explain plan should show if a Map/Reduce Work is being cached diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java index a633d2b913..7a7d97c8a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java @@ -235,6 +235,9 @@ private void combineEquivalentDPPSinks(SparkPartitionPruningSinkDesc first, } private void replaceWork(BaseWork previous, BaseWork current, SparkWork sparkWork) { + LOG.info("Combining equivalent work objects " + previous.getName() + " and " + current.getName() + + " for Spark DagName " + sparkWork.getName()); + updateReference(previous, current, sparkWork); List parents = sparkWork.getParents(previous); List children = sparkWork.getChildren(previous); @@ -256,6 +259,7 @@ private void replaceWork(BaseWork previous, BaseWork current, SparkWork sparkWor if (previous instanceof MapWork) { removedMapWorkNames.add(previous.getName()); } + current.setIsCombined(true); } /* diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java index ae7cd57920..01ffaebb96 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; @@ -108,6 +107,8 @@ public BaseWork(String name) { private Map inputSourceToRuntimeValuesInfo = new HashMap(); + private boolean isCombined; + public void setGatheringStats(boolean gatherStats) { this.gatheringStats = gatherStats; } @@ -490,4 +491,14 @@ public void setInputSourceToRuntimeValuesInfo( String workName, RuntimeValuesInfo runtimeValuesInfo) { inputSourceToRuntimeValuesInfo.put(workName, runtimeValuesInfo); } + + @Explain(displayName = "Is Combined", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, + displayOnlyOnTrue = true) + public boolean getIsCombined() { + return isCombined; + } + + public void setIsCombined(boolean isCombined) { + this.isCombined = isCombined; + } }