diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
index 16675f2..47b1c3d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
@@ -101,4 +101,9 @@ public String getName() {
public static String getOperatorName() {
return "COLLECT";
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
index a9f2218..6d7335c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
@@ -381,4 +381,9 @@ static public String getOperatorName() {
public OperatorType getType() {
return OperatorType.DEMUX;
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
index 8e516ce..648587c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
@@ -73,4 +73,9 @@ public ForwardOperator(CompilationOpContext ctx) {
protected void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java
index edc400a..23c3c2e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java
@@ -66,4 +66,9 @@ public LateralViewForwardOperator(CompilationOpContext ctx) {
protected void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
index 0633854..c2aebc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
@@ -112,4 +112,9 @@ public String getName() {
public static String getOperatorName() {
return "LIST_SINK";
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java
index 82d0017..4bfd964 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java
@@ -340,4 +340,9 @@ static public String getOperatorName() {
public OperatorType getType() {
return OperatorType.MUX;
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 66c34aa..73ddf86 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -1543,4 +1543,14 @@ public int getIndexForTezUnion() {
public void setIndexForTezUnion(int indexForTezUnion) {
this.indexForTezUnion = indexForTezUnion;
}
+
+ /**
+ * Decides whether two operators are logically the same.
+ * This can be used to merge same operators and avoid repeated computation.
+ */
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName()) &&
+ (conf == other.getConf() || (conf != null && other.getConf() != null &&
+ conf.isSame(other.getConf())));
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
index 99822a3..2a623d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
@@ -190,4 +190,9 @@ public boolean opAllowedBeforeSortMergeJoin() {
// it would be difficult to figure out the big table for the mapjoin.
return false;
}
+
+ @Override
+ public boolean logicalEquals(Operator other) {
+ return getClass().getName().equals(other.getClass().getName());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
deleted file mode 100644
index 0373e53..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
+++ /dev/null
@@ -1,614 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer;
-
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
-import org.apache.hadoop.hive.ql.exec.CollectOperator;
-import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
-import org.apache.hadoop.hive.ql.exec.DemuxOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.ForwardOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MuxOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.PTFOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ScriptOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.SparkHashTableSinkOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.TemporaryHashSinkOperator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator;
-import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
-import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
-import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc;
-import org.apache.hadoop.hive.ql.plan.GroupByDesc;
-import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc;
-import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
-import org.apache.hadoop.hive.ql.plan.LimitDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
-import org.apache.hadoop.hive.ql.plan.ScriptDesc;
-import org.apache.hadoop.hive.ql.plan.SelectDesc;
-import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-
-public class OperatorComparatorFactory {
- private static final Map, OperatorComparator> comparatorMapping = Maps.newHashMap();
- private static final Logger LOG = LoggerFactory.getLogger(OperatorComparatorFactory.class);
-
- static {
- comparatorMapping.put(TableScanOperator.class, new TableScanOperatorComparator());
- comparatorMapping.put(SelectOperator.class, new SelectOperatorComparator());
- comparatorMapping.put(FilterOperator.class, new FilterOperatorComparator());
- comparatorMapping.put(GroupByOperator.class, new GroupByOperatorComparator());
- comparatorMapping.put(ReduceSinkOperator.class, new ReduceSinkOperatorComparator());
- comparatorMapping.put(FileSinkOperator.class, new FileSinkOperatorComparator());
- comparatorMapping.put(JoinOperator.class, new JoinOperatorComparator());
- comparatorMapping.put(MapJoinOperator.class, new MapJoinOperatorComparator());
- comparatorMapping.put(SMBMapJoinOperator.class, new SMBMapJoinOperatorComparator());
- comparatorMapping.put(LimitOperator.class, new LimitOperatorComparator());
- comparatorMapping.put(SparkHashTableSinkOperator.class, new SparkHashTableSinkOperatorComparator());
- comparatorMapping.put(VectorSparkHashTableSinkOperator.class,
- new SparkHashTableSinkOperatorComparator());
- comparatorMapping.put(LateralViewJoinOperator.class, new LateralViewJoinOperatorComparator());
- comparatorMapping.put(VectorGroupByOperator.class, new VectorGroupByOperatorComparator());
- comparatorMapping.put(CommonMergeJoinOperator.class, new CommonMergeJoinOperatorComparator());
- comparatorMapping.put(VectorFilterOperator.class, new FilterOperatorComparator());
- comparatorMapping.put(UDTFOperator.class, new UDTFOperatorComparator());
- comparatorMapping.put(VectorSelectOperator.class, new VectorSelectOperatorComparator());
- comparatorMapping.put(VectorLimitOperator.class, new LimitOperatorComparator());
- comparatorMapping.put(ScriptOperator.class, new ScriptOperatorComparator());
- comparatorMapping.put(TemporaryHashSinkOperator.class, new HashTableSinkOperatorComparator());
- comparatorMapping.put(AppMasterEventOperator.class, new AppMasterEventOperatorComparator());
- // these operators does not have state, so they always equal with the same kind.
- comparatorMapping.put(UnionOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(ForwardOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(LateralViewForwardOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(DemuxOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(MuxOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(ListSinkOperator.class, new AlwaysTrueOperatorComparator());
- comparatorMapping.put(CollectOperator.class, new AlwaysTrueOperatorComparator());
- // do not support PTFOperator comparing now.
- comparatorMapping.put(PTFOperator.class, AlwaysFalseOperatorComparator.getInstance());
- }
-
- public static OperatorComparator getOperatorComparator(Class extends Operator> operatorClass) {
- OperatorComparator operatorComparator = comparatorMapping.get(operatorClass);
- if (operatorComparator == null) {
- LOG.warn("No OperatorComparator is registered for " + operatorClass.getName() +
- ". Default to always false comparator.");
- return AlwaysFalseOperatorComparator.getInstance();
- }
-
- return operatorComparator;
- }
-
- public interface OperatorComparator> {
- boolean equals(T op1, T op2);
- }
-
- static class AlwaysTrueOperatorComparator implements OperatorComparator> {
-
- @Override
- public boolean equals(Operator> op1, Operator> op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- return true;
- }
- }
-
- static class AlwaysFalseOperatorComparator implements OperatorComparator> {
- // the outer class is responsible for maintaining the comparator singleton
- private AlwaysFalseOperatorComparator() {
- }
-
- private static final AlwaysFalseOperatorComparator instance =
- new AlwaysFalseOperatorComparator();
-
- public static AlwaysFalseOperatorComparator getInstance() {
- return instance;
- }
-
- @Override
- public boolean equals(Operator> op1, Operator> op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- return false;
- }
- }
-
- static class TableScanOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(TableScanOperator op1, TableScanOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- TableScanDesc op1Conf = op1.getConf();
- TableScanDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getAlias(), op2Conf.getAlias()) &&
- compareExprNodeDesc(op1Conf.getFilterExpr(), op2Conf.getFilterExpr()) &&
- op1Conf.getRowLimit() == op2Conf.getRowLimit() &&
- op1Conf.isGatherStats() == op2Conf.isGatherStats()) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class SelectOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(SelectOperator op1, SelectOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- SelectDesc op1Conf = op1.getConf();
- SelectDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) &&
- compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) &&
- compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class VectorSelectOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(VectorSelectOperator op1, VectorSelectOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- SelectDesc op1Conf = op1.getConf();
- SelectDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) &&
- compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) &&
- compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class FilterOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(FilterOperator op1, FilterOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- FilterDesc op1Conf = op1.getConf();
- FilterDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getPredicateString(), op2Conf.getPredicateString()) &&
- (op1Conf.getIsSamplingPred() == op2Conf.getIsSamplingPred()) &&
- compareString(op1Conf.getSampleDescExpr(), op2Conf.getSampleDescExpr())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class GroupByOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(GroupByOperator op1, GroupByOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- GroupByDesc op1Conf = op1.getConf();
- GroupByDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getModeString(), op2Conf.getModeString()) &&
- compareString(op1Conf.getKeyString(), op2Conf.getKeyString()) &&
- compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) &&
- op1Conf.pruneGroupingSetId() == op2Conf.pruneGroupingSetId() &&
- compareObject(op1Conf.getAggregatorStrings(), op2Conf.getAggregatorStrings()) &&
- op1Conf.getBucketGroup() == op2Conf.getBucketGroup()) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class VectorGroupByOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(VectorGroupByOperator op1, VectorGroupByOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- GroupByDesc op1Conf = op1.getConf();
- GroupByDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getModeString(), op2Conf.getModeString()) &&
- compareString(op1Conf.getKeyString(), op2Conf.getKeyString()) &&
- compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) &&
- op1Conf.pruneGroupingSetId() == op2Conf.pruneGroupingSetId() &&
- compareObject(op1Conf.getAggregatorStrings(), op2Conf.getAggregatorStrings()) &&
- op1Conf.getBucketGroup() == op2Conf.getBucketGroup()) {
- return true;
- } else {
- return false;
- }
- }
- }
-
-
- static class ReduceSinkOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(ReduceSinkOperator op1, ReduceSinkOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- ReduceSinkDesc op1Conf = op1.getConf();
- ReduceSinkDesc op2Conf = op2.getConf();
-
- if (compareExprNodeDescList(op1Conf.getKeyCols(), op2Conf.getKeyCols()) &&
- compareExprNodeDescList(op1Conf.getValueCols(), op2Conf.getValueCols()) &&
- compareExprNodeDescList(op1Conf.getPartitionCols(), op2Conf.getPartitionCols()) &&
- op1Conf.getTag() == op2Conf.getTag() &&
- compareString(op1Conf.getOrder(), op2Conf.getOrder()) &&
- op1Conf.getTopN() == op2Conf.getTopN() &&
- op1Conf.isAutoParallel() == op2Conf.isAutoParallel()) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class FileSinkOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(FileSinkOperator op1, FileSinkOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- FileSinkDesc op1Conf = op1.getConf();
- FileSinkDesc op2Conf = op2.getConf();
-
- if (compareObject(op1Conf.getDirName(), op2Conf.getDirName()) &&
- compareObject(op1Conf.getTableInfo(), op2Conf.getTableInfo()) &&
- op1Conf.getCompressed() == op2Conf.getCompressed() &&
- op1Conf.getDestTableId() == op2Conf.getDestTableId() &&
- op1Conf.isMultiFileSpray() == op2Conf.isMultiFileSpray() &&
- op1Conf.getTotalFiles() == op2Conf.getTotalFiles() &&
- op1Conf.getNumFiles() == op2Conf.getNumFiles() &&
- compareString(op1Conf.getStaticSpec(), op2Conf.getStaticSpec()) &&
- op1Conf.isGatherStats() == op2Conf.isGatherStats() &&
- compareString(op1Conf.getStatsAggPrefix(), op2Conf.getStatsAggPrefix())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class JoinOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(JoinOperator op1, JoinOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- JoinDesc desc1 = op1.getConf();
- JoinDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class MapJoinOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(MapJoinOperator op1, MapJoinOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- MapJoinDesc desc1 = op1.getConf();
- MapJoinDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) &&
- compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- desc1.getPosBigTable() == desc2.getPosBigTable() &&
- desc1.isBucketMapJoin() == desc2.isBucketMapJoin() &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class CommonMergeJoinOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(CommonMergeJoinOperator op1, CommonMergeJoinOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- CommonMergeJoinDesc desc1 = op1.getConf();
- CommonMergeJoinDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) &&
- compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- desc1.getPosBigTable() == desc2.getPosBigTable() &&
- desc1.isBucketMapJoin() == desc2.isBucketMapJoin() &&
- desc1.getNumBuckets() == desc2.getNumBuckets() &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class SMBMapJoinOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(SMBMapJoinOperator op1, SMBMapJoinOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- SMBJoinDesc desc1 = op1.getConf();
- SMBJoinDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) &&
- compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- desc1.getPosBigTable() == desc2.getPosBigTable() &&
- desc1.isBucketMapJoin() == desc2.isBucketMapJoin() &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class LimitOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(LimitOperator op1, LimitOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- LimitDesc desc1 = op1.getConf();
- LimitDesc desc2 = op2.getConf();
-
- return desc1.getLimit() == desc2.getLimit();
- }
- }
-
- static class SparkHashTableSinkOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(SparkHashTableSinkOperator op1, SparkHashTableSinkOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- SparkHashTableSinkDesc desc1 = op1.getConf();
- SparkHashTableSinkDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getFilterMapString(), desc2.getFilterMapString()) &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- desc1.getPosBigTable() == desc2.getPosBigTable() &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class HashTableSinkOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(HashTableSinkOperator op1, HashTableSinkOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- HashTableSinkDesc desc1 = op1.getConf();
- HashTableSinkDesc desc2 = op2.getConf();
-
- if (compareObject(desc1.getFilterMapString(), desc2.getFilterMapString()) &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- desc1.getPosBigTable() == desc2.getPosBigTable() &&
- compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
- compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
- compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
- compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
- desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
- compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class LateralViewJoinOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(LateralViewJoinOperator op1, LateralViewJoinOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- LateralViewJoinDesc desc1 = op1.getConf();
- LateralViewJoinDesc desc2 = op2.getConf();
-
- return compareObject(desc1.getOutputInternalColNames(), desc2.getOutputInternalColNames());
- }
- }
-
- static class ScriptOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(ScriptOperator op1, ScriptOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- ScriptDesc desc1 = op1.getConf();
- ScriptDesc desc2 = op2.getConf();
-
- if (compareString(desc1.getScriptCmd(), desc2.getScriptCmd()) &&
- compareObject(desc1.getScriptOutputInfo(), desc2.getScriptOutputInfo())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class UDTFOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(UDTFOperator op1, UDTFOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- UDTFDesc desc1 = op1.getConf();
- UDTFDesc desc2 = op2.getConf();
-
- if (compareString(desc1.getUDTFName(), desc2.getUDTFName()) &&
- compareString(desc1.isOuterLateralView(), desc2.isOuterLateralView())) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- static class AppMasterEventOperatorComparator implements OperatorComparator {
-
- @Override
- public boolean equals(AppMasterEventOperator op1, AppMasterEventOperator op2) {
- Preconditions.checkNotNull(op1);
- Preconditions.checkNotNull(op2);
- AppMasterEventDesc op1Conf = op1.getConf();
- AppMasterEventDesc op2Conf = op2.getConf();
-
- if (compareString(op1Conf.getInputName(), op2Conf.getInputName()) &&
- compareString(op1Conf.getVertexName(), op2Conf.getVertexName()) &&
- compareObject(op1Conf.getTable(), op2Conf.getTable())) {
- if (op1Conf instanceof DynamicPruningEventDesc && op2Conf instanceof DynamicPruningEventDesc) {
- DynamicPruningEventDesc op1DPPConf = (DynamicPruningEventDesc) op1Conf;
- DynamicPruningEventDesc op2DPPConf = (DynamicPruningEventDesc) op2Conf;
- if (compareString(op1DPPConf.getTargetColumnName(), op2DPPConf.getTargetColumnName()) &&
- compareString(op1DPPConf.getTargetColumnType(), op2DPPConf.getTargetColumnType()) &&
- compareString(op1DPPConf.getPartKeyString(), op2DPPConf.getPartKeyString())) {
- return true;
- }
- return false;
- } else if (op1Conf instanceof DynamicPruningEventDesc || op2Conf instanceof DynamicPruningEventDesc) {
- return false;
- }
- return true;
- } else {
- return false;
- }
- }
- }
-
- static boolean compareString(String first, String second) {
- return compareObject(first, second);
- }
-
- /*
- * Compare Objects which implements its own meaningful equals methods.
- */
- static boolean compareObject(Object first, Object second) {
- return first == null ? second == null : first.equals(second);
- }
-
- static boolean compareExprNodeDesc(ExprNodeDesc first, ExprNodeDesc second) {
- return first == null ? second == null : first.isSame(second);
- }
-
- static boolean compareExprNodeDescList(List first, List second) {
- if (first == null && second == null) {
- return true;
- }
- if ((first == null && second != null) || (first != null && second == null)) {
- return false;
- }
- if (first.size() != second.size()) {
- return false;
- } else {
- for (int i = 0; i < first.size(); i++) {
- if (!first.get(i).isSame(second.get(i))) {
- return false;
- }
- }
- }
- return true;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index 8070c2a..b206ace 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -751,6 +751,7 @@ private static boolean compareOperator(ParseContext pctx, Operator> op1, Opera
// We can ignore table alias since when we compare ReduceSinkOperator, all
// its ancestors need to match (down to table scan), thus we make sure that
// both plans are the same.
+ // TODO: move this to logicalEquals
if (op1 instanceof ReduceSinkOperator) {
ReduceSinkDesc op1Conf = ((ReduceSinkOperator) op1).getConf();
ReduceSinkDesc op2Conf = ((ReduceSinkOperator) op2).getConf();
@@ -770,6 +771,7 @@ private static boolean compareOperator(ParseContext pctx, Operator> op1, Opera
// We handle TableScanOperator here as we can safely ignore table alias
// and the current comparator implementation does not.
+ // TODO: move this to logicalEquals
if (op1 instanceof TableScanOperator) {
TableScanOperator tsOp1 = (TableScanOperator) op1;
TableScanOperator tsOp2 = (TableScanOperator) op2;
@@ -790,9 +792,7 @@ private static boolean compareOperator(ParseContext pctx, Operator> op1, Opera
}
}
- OperatorComparatorFactory.OperatorComparator operatorComparator =
- OperatorComparatorFactory.getOperatorComparator(op1.getClass());
- return operatorComparator.equals(op1, op2);
+ return op1.logicalEquals(op2);
}
private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizerCache optimizerCache,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java
index ec192a0..95ad962 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java
@@ -40,7 +40,6 @@
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
-import org.apache.hadoop.hive.ql.optimizer.OperatorComparatorFactory;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalPlanResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -305,13 +304,7 @@ private boolean compareOperatorChain(Operator> firstOperator, Operator> seco
* @return
*/
private boolean compareCurrentOperator(Operator> firstOperator, Operator> secondOperator) {
- if (!firstOperator.getClass().getName().equals(secondOperator.getClass().getName())) {
- return false;
- }
-
- OperatorComparatorFactory.OperatorComparator operatorComparator =
- OperatorComparatorFactory.getOperatorComparator(firstOperator.getClass());
- return operatorComparator.equals(firstOperator, secondOperator);
+ return firstOperator.logicalEquals(secondOperator);
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
index fd46aba..66ee06a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
@@ -124,4 +124,13 @@ public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir) {
this.runtimeStatsTmpDir = runtimeStatsTmpDir;
}
+ /**
+ * The default implementation delegates to {@link #equals(Object)}. Intended to be
+ * overridden by sub classes.
+ */
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ return equals(other);
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java
index c5294f0..97fcd09 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.util.List;
+import java.util.Objects;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
@@ -84,4 +85,15 @@ public AppMasterEventOperatorExplainVectorization getAppMasterEventVectorization
}
return new AppMasterEventOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ AppMasterEventDesc otherDesc = (AppMasterEventDesc) other;
+ return Objects.equals(getInputName(), otherDesc.getInputName()) &&
+ Objects.equals(getVertexName(), otherDesc.getVertexName()) &&
+ Objects.equals(getTable(), otherDesc.getTable());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java
index cce9bc4..56683e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java
@@ -49,4 +49,13 @@ public int getBigTablePosition() {
public void setBigTablePosition(int pos) {
mapJoinConversionPos = pos;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (super.isSame(other)) {
+ CommonMergeJoinDesc otherDesc = (CommonMergeJoinDesc) other;
+ return getNumBuckets() == otherDesc.getNumBuckets();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
index d88e110..385c92a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.IOException;
+import java.util.Objects;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
@@ -100,4 +101,15 @@ public String getPartKeyString() {
public ExprNodeDesc getPartKey() {
return this.partKey;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (super.isSame(other)) {
+ DynamicPruningEventDesc otherDesc = (DynamicPruningEventDesc) other;
+ return Objects.equals(getTargetColumnName(), otherDesc.getTargetColumnName()) &&
+ Objects.equals(getTargetColumnType(), otherDesc.getTargetColumnType()) &&
+ Objects.equals(getPartKeyString(), otherDesc.getPartKeyString());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index df3de03..067fbe0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -915,4 +915,25 @@ public static ColumnOrigin findColumnOrigin(ExprNodeDesc expr, Operator> op) {
return findColumnOrigin(parentExpr, parentOp);
}
+
+ // Null-safe isSame
+ public static boolean isSame(ExprNodeDesc desc1, ExprNodeDesc desc2) {
+ return (desc1 == desc2) || (desc1 != null && desc1.isSame(desc2));
+ }
+
+ // Null-safe isSame for lists of ExprNodeDesc
+ public static boolean isSame(List first, List second) {
+ if (first == second) {
+ return true;
+ }
+ if (first == null || second == null || first.size() != second.size()) {
+ return false;
+ }
+ for (int i = 0; i < first.size(); i++) {
+ if (!first.get(i).isSame(second.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index fd27f53..a3df166 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -519,4 +520,22 @@ public void setInsertOverwrite(boolean isInsertOverwrite) {
public boolean getInsertOverwrite() {
return isInsertOverwrite;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ FileSinkDesc otherDesc = (FileSinkDesc) other;
+ return Objects.equals(getDirName(), otherDesc.getDirName()) &&
+ Objects.equals(getTableInfo(), otherDesc.getTableInfo()) &&
+ getCompressed() == otherDesc.getCompressed() &&
+ getDestTableId() == otherDesc.getDestTableId() &&
+ isMultiFileSpray() == otherDesc.isMultiFileSpray() &&
+ getTotalFiles() == otherDesc.getTotalFiles() &&
+ getNumFiles() == otherDesc.getNumFiles() &&
+ Objects.equals(getStaticSpec(), otherDesc.getStaticSpec()) &&
+ isGatherStats() == otherDesc.isGatherStats() &&
+ Objects.equals(getStatsAggPrefix(), otherDesc.getStatsAggPrefix());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
index e93660a..3de310c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
@@ -213,4 +214,15 @@ public FilterOperatorExplainVectorization getFilterVectorization() {
}
return new FilterOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ FilterDesc otherDesc = (FilterDesc) other;
+ return Objects.equals(getPredicateString(), otherDesc.getPredicateString()) &&
+ Objects.equals(getSampleDescExpr(), otherDesc.getSampleDescExpr()) &&
+ getIsSamplingPred() == otherDesc.getIsSamplingPred();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
index 45d100d..489a3b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
@@ -423,4 +424,18 @@ public static String getComplexTypeWithGroupByEnabledCondition(
HiveConf.ConfVars.HIVE_VECTORIZATION_GROUPBY_COMPLEX_TYPES_ENABLED.varname + " " + isVectorizationGroupByComplexTypesEnabled +
") IS " + enabled;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ GroupByDesc otherDesc = (GroupByDesc) other;
+ return Objects.equals(getModeString(), otherDesc.getModeString()) &&
+ Objects.equals(getKeyString(), otherDesc.getKeyString()) &&
+ Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) &&
+ pruneGroupingSetId() == otherDesc.pruneGroupingSetId() &&
+ Objects.equals(getAggregatorStrings(), otherDesc.getAggregatorStrings()) &&
+ getBucketGroup() == otherDesc.getBucketGroup();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
index 94ac41e..098ecb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
@@ -23,6 +23,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -389,4 +390,13 @@ public BucketMapJoinContext getBucketMapjoinContext() {
public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) {
this.bucketMapjoinContext = bucketMapjoinContext;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (super.isSame(other)) {
+ HashTableSinkDesc otherDesc = (HashTableSinkDesc) other;
+ return Objects.equals(getFilterMapString(), otherDesc.getFilterMapString());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index eae80a7..da9570d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -25,6 +25,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.MemoryMonitorInfo;
@@ -709,4 +710,18 @@ public long getInMemoryDataSize() {
public void setInMemoryDataSize(final long inMemoryDataSize) {
this.inMemoryDataSize = inMemoryDataSize;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ JoinDesc otherDesc = (JoinDesc) other;
+ return Objects.equals(getKeysString(), otherDesc.getKeysString()) &&
+ Objects.equals(getFiltersStringMap(), otherDesc.getFiltersStringMap()) &&
+ Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) &&
+ Objects.equals(getCondsList(), otherDesc.getCondsList()) &&
+ getHandleSkewJoin() == otherDesc.getHandleSkewJoin() &&
+ Objects.equals(getNullSafeString(), otherDesc.getNullSafeString());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
index d19cb3d..e8bae34 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
+import java.util.Objects;
+
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -63,4 +65,13 @@ public int getNumSelColumns() {
public void setNumSelColumns(int numSelColumns) {
this.numSelColumns = numSelColumns;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ LateralViewJoinDesc otherDesc = (LateralViewJoinDesc) other;
+ return Objects.equals(getOutputInternalColNames(), otherDesc.getOutputInternalColNames());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java
index b9cf337..952c586 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java
@@ -90,4 +90,13 @@ public LimitOperatorExplainVectorization getLimitVectorization() {
}
return new LimitOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ LimitDesc otherDesc = (LimitDesc) other;
+ return getLimit() == otherDesc.getLimit();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index e1b4ae6..1b5bd78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -28,6 +28,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -588,4 +589,16 @@ public SMBJoinOperatorExplainVectorization getSMBJoinVectorization() {
}
return new SMBJoinOperatorExplainVectorization((SMBJoinDesc) this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (super.isSame(other)) {
+ MapJoinDesc otherDesc = (MapJoinDesc) other;
+ return Objects.equals(getParentToInput(), otherDesc.getParentToInput()) &&
+ Objects.equals(getKeyCountsExplainDesc(), otherDesc.getKeyCountsExplainDesc()) &&
+ getPosBigTable() == otherDesc.getPosBigTable() &&
+ isBucketMapJoin() == otherDesc.isBucketMapJoin();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
index 850576c..6437172 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
@@ -34,4 +34,5 @@
public void setMaxMemoryAvailable(long memoryAvailble);
public String getRuntimeStatsTmpDir();
public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir);
+ boolean isSame(OperatorDesc other);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index d08e700..8820833 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -23,6 +23,7 @@
import java.util.EnumSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -648,4 +649,19 @@ public ReduceSinkOperatorExplainVectorization getReduceSinkVectorization() {
}
return new ReduceSinkOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ ReduceSinkDesc otherDesc = (ReduceSinkDesc) other;
+ return ExprNodeDescUtils.isSame(getKeyCols(), otherDesc.getKeyCols()) &&
+ ExprNodeDescUtils.isSame(getValueCols(), otherDesc.getValueCols()) &&
+ ExprNodeDescUtils.isSame(getPartitionCols(), otherDesc.getPartitionCols()) &&
+ getTag() == otherDesc.getTag() &&
+ Objects.equals(getOrder(), otherDesc.getOrder()) &&
+ getTopN() == otherDesc.getTopN() &&
+ isAutoParallel() == otherDesc.isAutoParallel();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java
index 5317894..b1c9da5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java
@@ -22,6 +22,8 @@
import org.apache.hadoop.hive.ql.exec.RecordWriter;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import java.util.Objects;
+
/**
* ScriptDesc.
@@ -143,4 +145,13 @@ public void setInRecordWriterClass(
this.inRecordWriterClass = inRecordWriterClass;
}
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ ScriptDesc otherDesc = (ScriptDesc) other;
+ return Objects.equals(getScriptCmd(), otherDesc.getScriptCmd()) &&
+ Objects.equals(getScriptOutputInfo(), otherDesc.getScriptOutputInfo());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java
index 0601ce0..fcfd911 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
@@ -170,4 +171,15 @@ public SelectOperatorExplainVectorization getSelectVectorization() {
}
return new SelectOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ SelectDesc otherDesc = (SelectDesc) other;
+ return Objects.equals(getColListString(), otherDesc.getColListString()) &&
+ Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) &&
+ Objects.equals(explainNoCompute(), otherDesc.explainNoCompute());
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index d1c8690..a88d061 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -24,6 +24,7 @@
import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -454,4 +455,16 @@ public void setVectorized(boolean vectorized) {
public boolean isVectorized() {
return vectorized;
}
+
+ @Override
+ public boolean isSame(OperatorDesc other) {
+ if (getClass().getName().equals(other.getClass().getName())) {
+ TableScanDesc otherDesc = (TableScanDesc) other;
+ return Objects.equals(getAlias(), otherDesc.getAlias()) &&
+ ExprNodeDescUtils.isSame(getFilterExpr(), otherDesc.getFilterExpr()) &&
+ getRowLimit() == otherDesc.getRowLimit() &&
+ isGatherStats() == otherDesc.isGatherStats();
+ }
+ return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java
index 68f289e..3b5bcea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java
@@ -21,6 +21,8 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import java.util.Objects;
+
/**
* All member variables should have a setters and getters of the form get