diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java index 16675f2..47b1c3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java @@ -101,4 +101,9 @@ public String getName() { public static String getOperatorName() { return "COLLECT"; } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java index a9f2218..6d7335c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java @@ -381,4 +381,9 @@ static public String getOperatorName() { public OperatorType getType() { return OperatorType.DEMUX; } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java index 8e516ce..648587c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java @@ -73,4 +73,9 @@ public ForwardOperator(CompilationOpContext ctx) { protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java index edc400a..23c3c2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewForwardOperator.java @@ -66,4 +66,9 @@ public LateralViewForwardOperator(CompilationOpContext ctx) { protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java index 0633854..c2aebc8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java @@ -112,4 +112,9 @@ public String getName() { public static String getOperatorName() { return "LIST_SINK"; } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java index 82d0017..4bfd964 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MuxOperator.java @@ -340,4 +340,9 @@ static public String getOperatorName() { public OperatorType getType() { return OperatorType.MUX; } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 66c34aa..27a472c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -1543,4 +1543,13 @@ public int getIndexForTezUnion() { public void setIndexForTezUnion(int indexForTezUnion) { this.indexForTezUnion = indexForTezUnion; } + + /** + * Decides whether two operators are logically the same. + * This can be used to merge same operators and avoid repeated computation. + */ + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()) && + conf.isSame(other.getConf()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java index 99822a3..2a623d5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java @@ -190,4 +190,9 @@ public boolean opAllowedBeforeSortMergeJoin() { // it would be difficult to figure out the big table for the mapjoin. return false; } + + @Override + public boolean logicalEquals(Operator other) { + return getClass().getName().equals(other.getClass().getName()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java deleted file mode 100644 index 0373e53..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java +++ /dev/null @@ -1,614 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer; - -import java.util.List; -import java.util.Map; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; -import org.apache.hadoop.hive.ql.exec.CollectOperator; -import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; -import org.apache.hadoop.hive.ql.exec.DemuxOperator; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.ForwardOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; -import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; -import org.apache.hadoop.hive.ql.exec.LimitOperator; -import org.apache.hadoop.hive.ql.exec.ListSinkOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.MuxOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.PTFOperator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; -import org.apache.hadoop.hive.ql.exec.ScriptOperator; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.SparkHashTableSinkOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.TemporaryHashSinkOperator; -import org.apache.hadoop.hive.ql.exec.UDTFOperator; -import org.apache.hadoop.hive.ql.exec.UnionOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator; -import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; -import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc; -import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.FileSinkDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.GroupByDesc; -import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc; -import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc; -import org.apache.hadoop.hive.ql.plan.LimitDesc; -import org.apache.hadoop.hive.ql.plan.MapJoinDesc; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; -import org.apache.hadoop.hive.ql.plan.ScriptDesc; -import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.plan.UDTFDesc; - -public class OperatorComparatorFactory { - private static final Map, OperatorComparator> comparatorMapping = Maps.newHashMap(); - private static final Logger LOG = LoggerFactory.getLogger(OperatorComparatorFactory.class); - - static { - comparatorMapping.put(TableScanOperator.class, new TableScanOperatorComparator()); - comparatorMapping.put(SelectOperator.class, new SelectOperatorComparator()); - comparatorMapping.put(FilterOperator.class, new FilterOperatorComparator()); - comparatorMapping.put(GroupByOperator.class, new GroupByOperatorComparator()); - comparatorMapping.put(ReduceSinkOperator.class, new ReduceSinkOperatorComparator()); - comparatorMapping.put(FileSinkOperator.class, new FileSinkOperatorComparator()); - comparatorMapping.put(JoinOperator.class, new JoinOperatorComparator()); - comparatorMapping.put(MapJoinOperator.class, new MapJoinOperatorComparator()); - comparatorMapping.put(SMBMapJoinOperator.class, new SMBMapJoinOperatorComparator()); - comparatorMapping.put(LimitOperator.class, new LimitOperatorComparator()); - comparatorMapping.put(SparkHashTableSinkOperator.class, new SparkHashTableSinkOperatorComparator()); - comparatorMapping.put(VectorSparkHashTableSinkOperator.class, - new SparkHashTableSinkOperatorComparator()); - comparatorMapping.put(LateralViewJoinOperator.class, new LateralViewJoinOperatorComparator()); - comparatorMapping.put(VectorGroupByOperator.class, new VectorGroupByOperatorComparator()); - comparatorMapping.put(CommonMergeJoinOperator.class, new CommonMergeJoinOperatorComparator()); - comparatorMapping.put(VectorFilterOperator.class, new FilterOperatorComparator()); - comparatorMapping.put(UDTFOperator.class, new UDTFOperatorComparator()); - comparatorMapping.put(VectorSelectOperator.class, new VectorSelectOperatorComparator()); - comparatorMapping.put(VectorLimitOperator.class, new LimitOperatorComparator()); - comparatorMapping.put(ScriptOperator.class, new ScriptOperatorComparator()); - comparatorMapping.put(TemporaryHashSinkOperator.class, new HashTableSinkOperatorComparator()); - comparatorMapping.put(AppMasterEventOperator.class, new AppMasterEventOperatorComparator()); - // these operators does not have state, so they always equal with the same kind. - comparatorMapping.put(UnionOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(ForwardOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(LateralViewForwardOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(DemuxOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(MuxOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(ListSinkOperator.class, new AlwaysTrueOperatorComparator()); - comparatorMapping.put(CollectOperator.class, new AlwaysTrueOperatorComparator()); - // do not support PTFOperator comparing now. - comparatorMapping.put(PTFOperator.class, AlwaysFalseOperatorComparator.getInstance()); - } - - public static OperatorComparator getOperatorComparator(Class operatorClass) { - OperatorComparator operatorComparator = comparatorMapping.get(operatorClass); - if (operatorComparator == null) { - LOG.warn("No OperatorComparator is registered for " + operatorClass.getName() + - ". Default to always false comparator."); - return AlwaysFalseOperatorComparator.getInstance(); - } - - return operatorComparator; - } - - public interface OperatorComparator> { - boolean equals(T op1, T op2); - } - - static class AlwaysTrueOperatorComparator implements OperatorComparator> { - - @Override - public boolean equals(Operator op1, Operator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - return true; - } - } - - static class AlwaysFalseOperatorComparator implements OperatorComparator> { - // the outer class is responsible for maintaining the comparator singleton - private AlwaysFalseOperatorComparator() { - } - - private static final AlwaysFalseOperatorComparator instance = - new AlwaysFalseOperatorComparator(); - - public static AlwaysFalseOperatorComparator getInstance() { - return instance; - } - - @Override - public boolean equals(Operator op1, Operator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - return false; - } - } - - static class TableScanOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(TableScanOperator op1, TableScanOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - TableScanDesc op1Conf = op1.getConf(); - TableScanDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getAlias(), op2Conf.getAlias()) && - compareExprNodeDesc(op1Conf.getFilterExpr(), op2Conf.getFilterExpr()) && - op1Conf.getRowLimit() == op2Conf.getRowLimit() && - op1Conf.isGatherStats() == op2Conf.isGatherStats()) { - return true; - } else { - return false; - } - } - } - - static class SelectOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(SelectOperator op1, SelectOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - SelectDesc op1Conf = op1.getConf(); - SelectDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) && - compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && - compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) { - return true; - } else { - return false; - } - } - } - - static class VectorSelectOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(VectorSelectOperator op1, VectorSelectOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - SelectDesc op1Conf = op1.getConf(); - SelectDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) && - compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && - compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) { - return true; - } else { - return false; - } - } - } - - static class FilterOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(FilterOperator op1, FilterOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - FilterDesc op1Conf = op1.getConf(); - FilterDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getPredicateString(), op2Conf.getPredicateString()) && - (op1Conf.getIsSamplingPred() == op2Conf.getIsSamplingPred()) && - compareString(op1Conf.getSampleDescExpr(), op2Conf.getSampleDescExpr())) { - return true; - } else { - return false; - } - } - } - - static class GroupByOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(GroupByOperator op1, GroupByOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - GroupByDesc op1Conf = op1.getConf(); - GroupByDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getModeString(), op2Conf.getModeString()) && - compareString(op1Conf.getKeyString(), op2Conf.getKeyString()) && - compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && - op1Conf.pruneGroupingSetId() == op2Conf.pruneGroupingSetId() && - compareObject(op1Conf.getAggregatorStrings(), op2Conf.getAggregatorStrings()) && - op1Conf.getBucketGroup() == op2Conf.getBucketGroup()) { - return true; - } else { - return false; - } - } - } - - static class VectorGroupByOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(VectorGroupByOperator op1, VectorGroupByOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - GroupByDesc op1Conf = op1.getConf(); - GroupByDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getModeString(), op2Conf.getModeString()) && - compareString(op1Conf.getKeyString(), op2Conf.getKeyString()) && - compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && - op1Conf.pruneGroupingSetId() == op2Conf.pruneGroupingSetId() && - compareObject(op1Conf.getAggregatorStrings(), op2Conf.getAggregatorStrings()) && - op1Conf.getBucketGroup() == op2Conf.getBucketGroup()) { - return true; - } else { - return false; - } - } - } - - - static class ReduceSinkOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(ReduceSinkOperator op1, ReduceSinkOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - ReduceSinkDesc op1Conf = op1.getConf(); - ReduceSinkDesc op2Conf = op2.getConf(); - - if (compareExprNodeDescList(op1Conf.getKeyCols(), op2Conf.getKeyCols()) && - compareExprNodeDescList(op1Conf.getValueCols(), op2Conf.getValueCols()) && - compareExprNodeDescList(op1Conf.getPartitionCols(), op2Conf.getPartitionCols()) && - op1Conf.getTag() == op2Conf.getTag() && - compareString(op1Conf.getOrder(), op2Conf.getOrder()) && - op1Conf.getTopN() == op2Conf.getTopN() && - op1Conf.isAutoParallel() == op2Conf.isAutoParallel()) { - return true; - } else { - return false; - } - } - } - - static class FileSinkOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(FileSinkOperator op1, FileSinkOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - FileSinkDesc op1Conf = op1.getConf(); - FileSinkDesc op2Conf = op2.getConf(); - - if (compareObject(op1Conf.getDirName(), op2Conf.getDirName()) && - compareObject(op1Conf.getTableInfo(), op2Conf.getTableInfo()) && - op1Conf.getCompressed() == op2Conf.getCompressed() && - op1Conf.getDestTableId() == op2Conf.getDestTableId() && - op1Conf.isMultiFileSpray() == op2Conf.isMultiFileSpray() && - op1Conf.getTotalFiles() == op2Conf.getTotalFiles() && - op1Conf.getNumFiles() == op2Conf.getNumFiles() && - compareString(op1Conf.getStaticSpec(), op2Conf.getStaticSpec()) && - op1Conf.isGatherStats() == op2Conf.isGatherStats() && - compareString(op1Conf.getStatsAggPrefix(), op2Conf.getStatsAggPrefix())) { - return true; - } else { - return false; - } - } - } - - static class JoinOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(JoinOperator op1, JoinOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - JoinDesc desc1 = op1.getConf(); - JoinDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getKeysString(), desc2.getKeysString()) && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class MapJoinOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(MapJoinOperator op1, MapJoinOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - MapJoinDesc desc1 = op1.getConf(); - MapJoinDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - desc1.getPosBigTable() == desc2.getPosBigTable() && - desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class CommonMergeJoinOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(CommonMergeJoinOperator op1, CommonMergeJoinOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - CommonMergeJoinDesc desc1 = op1.getConf(); - CommonMergeJoinDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - desc1.getPosBigTable() == desc2.getPosBigTable() && - desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && - desc1.getNumBuckets() == desc2.getNumBuckets() && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class SMBMapJoinOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(SMBMapJoinOperator op1, SMBMapJoinOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - SMBJoinDesc desc1 = op1.getConf(); - SMBJoinDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - desc1.getPosBigTable() == desc2.getPosBigTable() && - desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class LimitOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(LimitOperator op1, LimitOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - LimitDesc desc1 = op1.getConf(); - LimitDesc desc2 = op2.getConf(); - - return desc1.getLimit() == desc2.getLimit(); - } - } - - static class SparkHashTableSinkOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(SparkHashTableSinkOperator op1, SparkHashTableSinkOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - SparkHashTableSinkDesc desc1 = op1.getConf(); - SparkHashTableSinkDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getFilterMapString(), desc2.getFilterMapString()) && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - desc1.getPosBigTable() == desc2.getPosBigTable() && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class HashTableSinkOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(HashTableSinkOperator op1, HashTableSinkOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - HashTableSinkDesc desc1 = op1.getConf(); - HashTableSinkDesc desc2 = op2.getConf(); - - if (compareObject(desc1.getFilterMapString(), desc2.getFilterMapString()) && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - desc1.getPosBigTable() == desc2.getPosBigTable() && - compareObject(desc1.getKeysString(), desc2.getKeysString()) && - compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && - compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && - compareObject(desc1.getCondsList(), desc2.getCondsList()) && - desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && - compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { - return true; - } else { - return false; - } - } - } - - static class LateralViewJoinOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(LateralViewJoinOperator op1, LateralViewJoinOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - LateralViewJoinDesc desc1 = op1.getConf(); - LateralViewJoinDesc desc2 = op2.getConf(); - - return compareObject(desc1.getOutputInternalColNames(), desc2.getOutputInternalColNames()); - } - } - - static class ScriptOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(ScriptOperator op1, ScriptOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - ScriptDesc desc1 = op1.getConf(); - ScriptDesc desc2 = op2.getConf(); - - if (compareString(desc1.getScriptCmd(), desc2.getScriptCmd()) && - compareObject(desc1.getScriptOutputInfo(), desc2.getScriptOutputInfo())) { - return true; - } else { - return false; - } - } - } - - static class UDTFOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(UDTFOperator op1, UDTFOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - UDTFDesc desc1 = op1.getConf(); - UDTFDesc desc2 = op2.getConf(); - - if (compareString(desc1.getUDTFName(), desc2.getUDTFName()) && - compareString(desc1.isOuterLateralView(), desc2.isOuterLateralView())) { - return true; - } else { - return false; - } - } - } - - static class AppMasterEventOperatorComparator implements OperatorComparator { - - @Override - public boolean equals(AppMasterEventOperator op1, AppMasterEventOperator op2) { - Preconditions.checkNotNull(op1); - Preconditions.checkNotNull(op2); - AppMasterEventDesc op1Conf = op1.getConf(); - AppMasterEventDesc op2Conf = op2.getConf(); - - if (compareString(op1Conf.getInputName(), op2Conf.getInputName()) && - compareString(op1Conf.getVertexName(), op2Conf.getVertexName()) && - compareObject(op1Conf.getTable(), op2Conf.getTable())) { - if (op1Conf instanceof DynamicPruningEventDesc && op2Conf instanceof DynamicPruningEventDesc) { - DynamicPruningEventDesc op1DPPConf = (DynamicPruningEventDesc) op1Conf; - DynamicPruningEventDesc op2DPPConf = (DynamicPruningEventDesc) op2Conf; - if (compareString(op1DPPConf.getTargetColumnName(), op2DPPConf.getTargetColumnName()) && - compareString(op1DPPConf.getTargetColumnType(), op2DPPConf.getTargetColumnType()) && - compareString(op1DPPConf.getPartKeyString(), op2DPPConf.getPartKeyString())) { - return true; - } - return false; - } else if (op1Conf instanceof DynamicPruningEventDesc || op2Conf instanceof DynamicPruningEventDesc) { - return false; - } - return true; - } else { - return false; - } - } - } - - static boolean compareString(String first, String second) { - return compareObject(first, second); - } - - /* - * Compare Objects which implements its own meaningful equals methods. - */ - static boolean compareObject(Object first, Object second) { - return first == null ? second == null : first.equals(second); - } - - static boolean compareExprNodeDesc(ExprNodeDesc first, ExprNodeDesc second) { - return first == null ? second == null : first.isSame(second); - } - - static boolean compareExprNodeDescList(List first, List second) { - if (first == null && second == null) { - return true; - } - if ((first == null && second != null) || (first != null && second == null)) { - return false; - } - if (first.size() != second.size()) { - return false; - } else { - for (int i = 0; i < first.size(); i++) { - if (!first.get(i).isSame(second.get(i))) { - return false; - } - } - } - return true; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java index 8070c2a..b206ace 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java @@ -751,6 +751,7 @@ private static boolean compareOperator(ParseContext pctx, Operator op1, Opera // We can ignore table alias since when we compare ReduceSinkOperator, all // its ancestors need to match (down to table scan), thus we make sure that // both plans are the same. + // TODO: move this to logicalEquals if (op1 instanceof ReduceSinkOperator) { ReduceSinkDesc op1Conf = ((ReduceSinkOperator) op1).getConf(); ReduceSinkDesc op2Conf = ((ReduceSinkOperator) op2).getConf(); @@ -770,6 +771,7 @@ private static boolean compareOperator(ParseContext pctx, Operator op1, Opera // We handle TableScanOperator here as we can safely ignore table alias // and the current comparator implementation does not. + // TODO: move this to logicalEquals if (op1 instanceof TableScanOperator) { TableScanOperator tsOp1 = (TableScanOperator) op1; TableScanOperator tsOp2 = (TableScanOperator) op2; @@ -790,9 +792,7 @@ private static boolean compareOperator(ParseContext pctx, Operator op1, Opera } } - OperatorComparatorFactory.OperatorComparator operatorComparator = - OperatorComparatorFactory.getOperatorComparator(op1.getClass()); - return operatorComparator.equals(op1, op2); + return op1.logicalEquals(op2); } private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java index ec192a0..95ad962 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/CombineEquivalentWorkResolver.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; -import org.apache.hadoop.hive.ql.optimizer.OperatorComparatorFactory; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalPlanResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -305,13 +304,7 @@ private boolean compareOperatorChain(Operator firstOperator, Operator seco * @return */ private boolean compareCurrentOperator(Operator firstOperator, Operator secondOperator) { - if (!firstOperator.getClass().getName().equals(secondOperator.getClass().getName())) { - return false; - } - - OperatorComparatorFactory.OperatorComparator operatorComparator = - OperatorComparatorFactory.getOperatorComparator(firstOperator.getClass()); - return operatorComparator.equals(firstOperator, secondOperator); + return firstOperator.logicalEquals(secondOperator); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java index fd46aba..66ee06a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java @@ -124,4 +124,13 @@ public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir) { this.runtimeStatsTmpDir = runtimeStatsTmpDir; } + /** + * The default implementation delegates to {@link #equals(Object)}. Intended to be + * overridden by sub classes. + */ + @Override + public boolean isSame(OperatorDesc other) { + return equals(other); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java index c5294f0..97fcd09 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; @@ -84,4 +85,15 @@ public AppMasterEventOperatorExplainVectorization getAppMasterEventVectorization } return new AppMasterEventOperatorExplainVectorization(this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + AppMasterEventDesc otherDesc = (AppMasterEventDesc) other; + return Objects.equals(getInputName(), otherDesc.getInputName()) && + Objects.equals(getVertexName(), otherDesc.getVertexName()) && + Objects.equals(getTable(), otherDesc.getTable()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java index cce9bc4..56683e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java @@ -49,4 +49,13 @@ public int getBigTablePosition() { public void setBigTablePosition(int pos) { mapJoinConversionPos = pos; } + + @Override + public boolean isSame(OperatorDesc other) { + if (super.isSame(other)) { + CommonMergeJoinDesc otherDesc = (CommonMergeJoinDesc) other; + return getNumBuckets() == otherDesc.getNumBuckets(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java index d88e110..385c92a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.io.IOException; +import java.util.Objects; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -100,4 +101,15 @@ public String getPartKeyString() { public ExprNodeDesc getPartKey() { return this.partKey; } + + @Override + public boolean isSame(OperatorDesc other) { + if (super.isSame(other)) { + DynamicPruningEventDesc otherDesc = (DynamicPruningEventDesc) other; + return Objects.equals(getTargetColumnName(), otherDesc.getTargetColumnName()) && + Objects.equals(getTargetColumnType(), otherDesc.getTargetColumnType()) && + Objects.equals(getPartKeyString(), otherDesc.getPartKeyString()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index df3de03..067fbe0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -915,4 +915,25 @@ public static ColumnOrigin findColumnOrigin(ExprNodeDesc expr, Operator op) { return findColumnOrigin(parentExpr, parentOp); } + + // Null-safe isSame + public static boolean isSame(ExprNodeDesc desc1, ExprNodeDesc desc2) { + return (desc1 == desc2) || (desc1 != null && desc1.isSame(desc2)); + } + + // Null-safe isSame for lists of ExprNodeDesc + public static boolean isSame(List first, List second) { + if (first == second) { + return true; + } + if (first == null || second == null || first.size() != second.size()) { + return false; + } + for (int i = 0; i < first.size(); i++) { + if (!first.get(i).isSame(second.get(i))) { + return false; + } + } + return true; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index fd27f53..a3df166 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -519,4 +520,22 @@ public void setInsertOverwrite(boolean isInsertOverwrite) { public boolean getInsertOverwrite() { return isInsertOverwrite; } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + FileSinkDesc otherDesc = (FileSinkDesc) other; + return Objects.equals(getDirName(), otherDesc.getDirName()) && + Objects.equals(getTableInfo(), otherDesc.getTableInfo()) && + getCompressed() == otherDesc.getCompressed() && + getDestTableId() == otherDesc.getDestTableId() && + isMultiFileSpray() == otherDesc.isMultiFileSpray() && + getTotalFiles() == otherDesc.getTotalFiles() && + getNumFiles() == otherDesc.getNumFiles() && + Objects.equals(getStaticSpec(), otherDesc.getStaticSpec()) && + isGatherStats() == otherDesc.isGatherStats() && + Objects.equals(getStatsAggPrefix(), otherDesc.getStatsAggPrefix()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index e93660a..3de310c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Objects; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; @@ -213,4 +214,15 @@ public FilterOperatorExplainVectorization getFilterVectorization() { } return new FilterOperatorExplainVectorization(this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + FilterDesc otherDesc = (FilterDesc) other; + return Objects.equals(getPredicateString(), otherDesc.getPredicateString()) && + Objects.equals(getSampleDescExpr(), otherDesc.getSampleDescExpr()) && + getIsSamplingPred() == otherDesc.getIsSamplingPred(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 45d100d..489a3b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; @@ -423,4 +424,18 @@ public static String getComplexTypeWithGroupByEnabledCondition( HiveConf.ConfVars.HIVE_VECTORIZATION_GROUPBY_COMPLEX_TYPES_ENABLED.varname + " " + isVectorizationGroupByComplexTypesEnabled + ") IS " + enabled; } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + GroupByDesc otherDesc = (GroupByDesc) other; + return Objects.equals(getModeString(), otherDesc.getModeString()) && + Objects.equals(getKeyString(), otherDesc.getKeyString()) && + Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) && + pruneGroupingSetId() == otherDesc.pruneGroupingSetId() && + Objects.equals(getAggregatorStrings(), otherDesc.getAggregatorStrings()) && + getBucketGroup() == otherDesc.getBucketGroup(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 94ac41e..098ecb1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -389,4 +390,13 @@ public BucketMapJoinContext getBucketMapjoinContext() { public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) { this.bucketMapjoinContext = bucketMapjoinContext; } + + @Override + public boolean isSame(OperatorDesc other) { + if (super.isSame(other)) { + HashTableSinkDesc otherDesc = (HashTableSinkDesc) other; + return Objects.equals(getFilterMapString(), otherDesc.getFilterMapString()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index eae80a7..da9570d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -25,6 +25,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.MemoryMonitorInfo; @@ -709,4 +710,18 @@ public long getInMemoryDataSize() { public void setInMemoryDataSize(final long inMemoryDataSize) { this.inMemoryDataSize = inMemoryDataSize; } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + JoinDesc otherDesc = (JoinDesc) other; + return Objects.equals(getKeysString(), otherDesc.getKeysString()) && + Objects.equals(getFiltersStringMap(), otherDesc.getFiltersStringMap()) && + Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) && + Objects.equals(getCondsList(), otherDesc.getCondsList()) && + getHandleSkewJoin() == otherDesc.getHandleSkewJoin() && + Objects.equals(getNullSafeString(), otherDesc.getNullSafeString()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java index d19cb3d..e8bae34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.plan; import java.util.ArrayList; +import java.util.Objects; + import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -63,4 +65,13 @@ public int getNumSelColumns() { public void setNumSelColumns(int numSelColumns) { this.numSelColumns = numSelColumns; } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + LateralViewJoinDesc otherDesc = (LateralViewJoinDesc) other; + return Objects.equals(getOutputInternalColNames(), otherDesc.getOutputInternalColNames()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java index b9cf337..952c586 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java @@ -90,4 +90,13 @@ public LimitOperatorExplainVectorization getLimitVectorization() { } return new LimitOperatorExplainVectorization(this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + LimitDesc otherDesc = (LimitDesc) other; + return getLimit() == otherDesc.getLimit(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index e1b4ae6..1b5bd78 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; @@ -588,4 +589,16 @@ public SMBJoinOperatorExplainVectorization getSMBJoinVectorization() { } return new SMBJoinOperatorExplainVectorization((SMBJoinDesc) this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (super.isSame(other)) { + MapJoinDesc otherDesc = (MapJoinDesc) other; + return Objects.equals(getParentToInput(), otherDesc.getParentToInput()) && + Objects.equals(getKeyCountsExplainDesc(), otherDesc.getKeyCountsExplainDesc()) && + getPosBigTable() == otherDesc.getPosBigTable() && + isBucketMapJoin() == otherDesc.isBucketMapJoin(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index 850576c..6437172 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -34,4 +34,5 @@ public void setMaxMemoryAvailable(long memoryAvailble); public String getRuntimeStatsTmpDir(); public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir); + boolean isSame(OperatorDesc other); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index d08e700..8820833 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -23,6 +23,7 @@ import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; @@ -648,4 +649,19 @@ public ReduceSinkOperatorExplainVectorization getReduceSinkVectorization() { } return new ReduceSinkOperatorExplainVectorization(this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + ReduceSinkDesc otherDesc = (ReduceSinkDesc) other; + return ExprNodeDescUtils.isSame(getKeyCols(), otherDesc.getKeyCols()) && + ExprNodeDescUtils.isSame(getValueCols(), otherDesc.getValueCols()) && + ExprNodeDescUtils.isSame(getPartitionCols(), otherDesc.getPartitionCols()) && + getTag() == otherDesc.getTag() && + Objects.equals(getOrder(), otherDesc.getOrder()) && + getTopN() == otherDesc.getTopN() && + isAutoParallel() == otherDesc.isAutoParallel(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java index 5317894..b1c9da5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java @@ -22,6 +22,8 @@ import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.plan.Explain.Level; +import java.util.Objects; + /** * ScriptDesc. @@ -143,4 +145,13 @@ public void setInRecordWriterClass( this.inRecordWriterClass = inRecordWriterClass; } + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + ScriptDesc otherDesc = (ScriptDesc) other; + return Objects.equals(getScriptCmd(), otherDesc.getScriptCmd()) && + Objects.equals(getScriptOutputInfo(), otherDesc.getScriptOutputInfo()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index 0601ce0..fcfd911 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; @@ -170,4 +171,15 @@ public SelectOperatorExplainVectorization getSelectVectorization() { } return new SelectOperatorExplainVectorization(this, vectorDesc); } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + SelectDesc otherDesc = (SelectDesc) other; + return Objects.equals(getColListString(), otherDesc.getColListString()) && + Objects.equals(getOutputColumnNames(), otherDesc.getOutputColumnNames()) && + Objects.equals(explainNoCompute(), otherDesc.explainNoCompute()); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index d1c8690..a88d061 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -24,6 +24,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Table; @@ -454,4 +455,16 @@ public void setVectorized(boolean vectorized) { public boolean isVectorized() { return vectorized; } + + @Override + public boolean isSame(OperatorDesc other) { + if (getClass().getName().equals(other.getClass().getName())) { + TableScanDesc otherDesc = (TableScanDesc) other; + return Objects.equals(getAlias(), otherDesc.getAlias()) && + ExprNodeDescUtils.isSame(getFilterExpr(), otherDesc.getFilterExpr()) && + getRowLimit() == otherDesc.getRowLimit() && + isGatherStats() == otherDesc.isGatherStats(); + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java index 68f289e..3b5bcea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java @@ -21,6 +21,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; import org.apache.hadoop.hive.ql.plan.Explain.Level; +import java.util.Objects; + /** * All member variables should have a setters and getters of the form get