diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java index 0373e53..293bcb9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java @@ -191,7 +191,7 @@ public boolean equals(SelectOperator op1, SelectOperator op2) { SelectDesc op1Conf = op1.getConf(); SelectDesc op2Conf = op2.getConf(); - if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) && + if (compareString(op1Conf.getOperatorSignature(), op2Conf.getOperatorSignature()) && compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) { return true; @@ -210,7 +210,7 @@ public boolean equals(VectorSelectOperator op1, VectorSelectOperator op2) { SelectDesc op1Conf = op1.getConf(); SelectDesc op2Conf = op2.getConf(); - if (compareString(op1Conf.getColListString(), op2Conf.getColListString()) && + if (compareString(op1Conf.getOperatorSignature(), op2Conf.getOperatorSignature()) && compareObject(op1Conf.getOutputColumnNames(), op2Conf.getOutputColumnNames()) && compareString(op1Conf.explainNoCompute(), op2Conf.explainNoCompute())) { return true; @@ -229,7 +229,7 @@ public boolean equals(FilterOperator op1, FilterOperator op2) { FilterDesc op1Conf = op1.getConf(); FilterDesc op2Conf = op2.getConf(); - if (compareString(op1Conf.getPredicateString(), op2Conf.getPredicateString()) && + if (compareString(op1Conf.getOperatorSignature(), op2Conf.getOperatorSignature()) && (op1Conf.getIsSamplingPred() == op2Conf.getIsSamplingPred()) && compareString(op1Conf.getSampleDescExpr(), op2Conf.getSampleDescExpr())) { return true; @@ -365,7 +365,7 @@ public boolean equals(MapJoinOperator op1, MapJoinOperator op2) { MapJoinDesc desc2 = op2.getConf(); if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && + compareString(desc1.getOperatorSignature(), desc2.getOperatorSignature()) && compareObject(desc1.getKeysString(), desc2.getKeysString()) && desc1.getPosBigTable() == desc2.getPosBigTable() && desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && @@ -391,7 +391,7 @@ public boolean equals(CommonMergeJoinOperator op1, CommonMergeJoinOperator op2) CommonMergeJoinDesc desc2 = op2.getConf(); if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && + compareString(desc1.getOperatorSignature(), desc2.getOperatorSignature()) && compareObject(desc1.getKeysString(), desc2.getKeysString()) && desc1.getPosBigTable() == desc2.getPosBigTable() && desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && @@ -418,7 +418,7 @@ public boolean equals(SMBMapJoinOperator op1, SMBMapJoinOperator op2) { SMBJoinDesc desc2 = op2.getConf(); if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && - compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && + compareString(desc1.getOperatorSignature(), desc2.getOperatorSignature()) && compareObject(desc1.getKeysString(), desc2.getKeysString()) && desc1.getPosBigTable() == desc2.getPosBigTable() && desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java index 8070c2a..61d4f0e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java @@ -776,14 +776,13 @@ private static boolean compareOperator(ParseContext pctx, Operator op1, Opera TableScanDesc op1Conf = tsOp1.getConf(); TableScanDesc op2Conf = tsOp2.getConf(); - if (StringUtils.equals( - op1Conf.getTableMetadata().getDbName() + "." + op1Conf.getTableMetadata().getTableName(), - op2Conf.getTableMetadata().getDbName() + "." + op2Conf.getTableMetadata().getTableName()) && - op1Conf.getNeededColumns().equals(op2Conf.getNeededColumns()) && - StringUtils.equals(op1Conf.getFilterExprString(), op2Conf.getFilterExprString()) && - pctx.getPrunedPartitions(tsOp1).getPartitions().equals( + if (StringUtils.equals(op1Conf.getTableMetadata().getDbName(), op2Conf.getTableMetadata().getDbName()) && + StringUtils.equals( op1Conf.getTableMetadata().getTableName(), op2Conf.getTableMetadata().getTableName()) && + op1Conf.getNeededColumns().equals(op2Conf.getNeededColumns()) && + StringUtils.equals(op1Conf.getFilterExprString(), op2Conf.getFilterExprString()) && + pctx.getPrunedPartitions(tsOp1).getPartitions().equals( pctx.getPrunedPartitions(tsOp2).getPartitions()) && - op1Conf.getRowLimit() == op2Conf.getRowLimit()) { + op1Conf.getRowLimit() == op2Conf.getRowLimit()) { return true; } else { return false; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java index fd46aba..aa41827 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java @@ -39,6 +39,7 @@ protected long memNeeded = 0; protected long memAvailable = 0; protected String runtimeStatsTmpDir; + protected transient String opSignature; @Override @Explain(skipHeader = true, displayName = "Statistics") @@ -124,4 +125,8 @@ public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir) { this.runtimeStatsTmpDir = runtimeStatsTmpDir; } + @Override + public String getOperatorSignature() { + return Integer.toString(this.hashCode()); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index e93660a..7ec113d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -108,6 +108,14 @@ public FilterDesc( this.sampleDescr = sampleDescr; } + @Override + public String getOperatorSignature() { + if (this.opSignature == null) { + this.opSignature = getPredicateString(); + } + return opSignature; + } + @Explain(displayName = "predicate") public String getPredicateString() { return PlanUtils.getExprListString(Arrays.asList(predicate)); @@ -125,6 +133,7 @@ public String getUserLevelExplainPredicateString() { public void setPredicate( final org.apache.hadoop.hive.ql.plan.ExprNodeDesc predicate) { this.predicate = predicate; + this.opSignature = null; } @Explain(displayName = "isSamplingPred", explainLevels = { Level.EXTENDED }) @@ -185,6 +194,7 @@ public Object clone() { filterDesc.setSampleDescr(getSampleDescr()); } filterDesc.setSortedFilter(isSortedFilter()); + filterDesc.opSignature = this.opSignature; return filterDesc; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index 0d8e459..73d4f36 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -107,6 +107,7 @@ public MapJoinDesc(MapJoinDesc clone) { this.parentDataSizes = clone.parentDataSizes; this.isBucketMapJoin = clone.isBucketMapJoin; this.isHybridHashJoin = clone.isHybridHashJoin; + this.opSignature = clone.opSignature; } public MapJoinDesc(final Map> keys, @@ -149,6 +150,7 @@ private void initRetainExprList() { public void setParentToInput(Map parentToInput) { this.parentToInput = parentToInput; + this.opSignature = null; } public Map getParentKeyCounts() { @@ -159,6 +161,14 @@ public void setParentToInput(Map parentToInput) { return parentDataSizes; } + @Override + public String getOperatorSignature() { + if (this.opSignature == null) { + this.opSignature = getKeyCountsExplainDesc(); + } + return this.opSignature; + } + @Explain(displayName = "Estimated key counts", explainLevels = { Level.EXTENDED }) public String getKeyCountsExplainDesc() { StringBuilder result = null; @@ -175,6 +185,7 @@ public String getKeyCountsExplainDesc() { public void setParentKeyCount(Map parentKeyCounts) { this.parentKeyCounts = parentKeyCounts; + this.opSignature = null; } public Map getValueIndices() { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index 850576c..4b725d8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -34,4 +34,5 @@ public void setMaxMemoryAvailable(long memoryAvailble); public String getRuntimeStatsTmpDir(); public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir); + public String getOperatorSignature(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index 0601ce0..431879e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -68,9 +68,18 @@ public Object clone() { new ArrayList(getOutputColumnNames())); ret.setSelectStar(selectStar); ret.setSelStarNoCompute(selStarNoCompute); + ret.opSignature = this.opSignature; return ret; } + @Override + public String getOperatorSignature() { + if (this.opSignature == null) { + this.opSignature = getColListString(); + } + return opSignature; + } + @Explain(displayName = "expressions") public String getColListString() { return PlanUtils.getExprListString(colList); @@ -83,6 +92,7 @@ public String getColListString() { public void setColList( final List colList) { this.colList = colList; + this.opSignature = null; } @Explain(displayName = "outputColumnNames")