diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PreCBOOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PreCBOOptimizer.java deleted file mode 100644 index e5b64a6..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PreCBOOptimizer.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.optimizer.lineage.Generator; -import org.apache.hadoop.hive.ql.optimizer.pcr.PartitionConditionRemover; -import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; -import org.apache.hadoop.hive.ql.optimizer.stats.annotation.AnnotateWithStatistics; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; -import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; - -/* - * do PredicatePushdown, PartitionPruning and Column Pruning before CBO - */ -public class PreCBOOptimizer { - private ParseContext pctx; - private List transformations; - - /** - * Create the list of transformations. - * - * @param hiveConf - */ - public void initialize(HiveConf hiveConf) { - transformations = new ArrayList(); - // Add the transformation that computes the lineage information. - transformations.add(new Generator()); - transformations.add(new PredicateTransitivePropagate()); - transformations.add(new PredicatePushDown()); - transformations.add(new PartitionPruner()); - transformations.add(new PartitionConditionRemover()); - transformations.add(new ColumnPruner()); - transformations.add(new AnnotateWithStatistics()); - } - - /** - * Invoke all the transformations one-by-one, and alter the query plan. - * - * @return ParseContext - * @throws SemanticException - */ - public ParseContext optimize() throws SemanticException { - for (Transform t : transformations) { - pctx = t.transform(pctx); - } - return pctx; - } - - /** - * @return the pctx - */ - public ParseContext getPctx() { - return pctx; - } - - /** - * @param pctx - * the pctx to set - */ - public void setPctx(ParseContext pctx) { - this.pctx = pctx; - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java index d30e697..e9e052f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq; import com.google.common.collect.ImmutableList; @@ -10,14 +27,6 @@ import org.eigenbase.rel.metadata.DefaultRelMetadataProvider; import org.eigenbase.rel.metadata.RelMetadataProvider; -/** - * Distinct row count and Selectivity is overridden for Hive.
- *

- * Distinct Row Count is overridden for:
- * 1) Join 2) TableScan.
- * Selectivity is overridden for:
- * 1) Join 2) TableScan & Filter. - */ public class HiveDefaultRelMetadataProvider { private HiveDefaultRelMetadataProvider() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index 9faae39..dca1949 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -1,20 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq; import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; +import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; import org.eigenbase.rel.RelFactories.ProjectFactory; import org.eigenbase.rel.RelNode; +import org.eigenbase.relopt.RelOptUtil; +import org.eigenbase.relopt.RelOptUtil.InputReferencedVisitor; import org.eigenbase.reltype.RelDataTypeField; import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; +import org.eigenbase.sql.SqlKind; import org.eigenbase.sql.fun.SqlStdOperatorTable; import org.eigenbase.sql.validate.SqlValidatorUtil; import org.eigenbase.util.Pair; import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; /** @@ -26,7 +55,7 @@ /** * Get list of virtual columns from the given list of projections. *

- * + * * @param exps * list of rex nodes representing projections * @return List of Virtual Columns, will not be null. @@ -66,14 +95,10 @@ public RexNode apply(RelDataTypeField field) { return projIndxLst; } - @Deprecated - public static void todo(String s) { - } - /** * Push any equi join conditions that are not column references as Projections * on top of the children. - * + * * @param factory * Project factory to use. * @param inputRels @@ -85,16 +110,17 @@ public static void todo(String s) { * @param systemColCount * number of system columns, usually zero. These columns are * projected at the leading edge of the output row. - * @param leftKeys on return this contains the join key positions from - * the new project rel on the LHS. - * @param rightKeys on return this contains the join key positions from - * the new project rel on the RHS. + * @param leftKeys + * on return this contains the join key positions from the new + * project rel on the LHS. + * @param rightKeys + * on return this contains the join key positions from the new + * project rel on the RHS. * @return the join condition after the equi expressions pushed down. */ - public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, - RelNode[] inputRels, List leftJoinKeys, - List rightJoinKeys, int systemColCount, List leftKeys, - List rightKeys) { + public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, RelNode[] inputRels, + List leftJoinKeys, List rightJoinKeys, int systemColCount, + List leftKeys, List rightKeys) { RelNode leftRel = inputRels[0]; RelNode rightRel = inputRels[1]; RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder(); @@ -118,8 +144,7 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, } for (i = 0; i < origRightInputSize; i++) { - final RelDataTypeField field = rightRel.getRowType().getFieldList() - .get(i); + final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i); newRightFields.add(rexBuilder.makeInputRef(field.getType(), i)); newRightFieldNames.add(field.getName()); } @@ -151,13 +176,12 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, RexNode cond = rexBuilder.makeCall( SqlStdOperatorTable.EQUALS, rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left), - rexBuilder.makeInputRef(rightKey.getType(), systemColCount - + origLeftInputSize + newKeyCount + p.right)); + rexBuilder.makeInputRef(rightKey.getType(), systemColCount + origLeftInputSize + + newKeyCount + p.right)); if (outJoinCond == null) { outJoinCond = cond; } else { - outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, - cond); + outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond); } } @@ -166,20 +190,17 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, } int newLeftOffset = systemColCount + origLeftInputSize; - int newRightOffset = systemColCount + origLeftInputSize - + origRightInputSize + newKeyCount; + int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount; for (i = 0; i < newKeyCount; i++) { leftKeys.add(origLeftInputSize + i); rightKeys.add(origRightInputSize + i); - RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexBuilder - .makeInputRef(newLeftFields.get(i).getType(), newLeftOffset + i), - rexBuilder.makeInputRef(newLeftFields.get(i).getType(), - newRightOffset + i)); + RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + rexBuilder.makeInputRef(newLeftFields.get(i).getType(), newLeftOffset + i), + rexBuilder.makeInputRef(newLeftFields.get(i).getType(), newRightOffset + i)); if (outJoinCond == null) { outJoinCond = cond; } else { - outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, - cond); + outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond); } } @@ -197,4 +218,260 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, return outJoinCond; } + + /** + * JoinPredicateInfo represents Join condition; JoinPredicate Info uses + * JoinLeafPredicateInfo to represent individual conjunctive elements in the + * predicate.
+ * JoinPredicateInfo = JoinLeafPredicateInfo1 and JoinLeafPredicateInfo2...
+ *

+ * JoinPredicateInfo:
+ * 1. preserves the order of conjuctive elements for + * equi-join(m_equiJoinPredicateElements)
+ * 2. Stores set of projection indexes from left and right child which is part + * of equi join keys; the indexes are both in child and Join node schema.
+ * 3. Keeps a map of projection indexes that are part of join keys to list of + * conjuctive elements(JoinLeafPredicateInfo) that uses them. + * + */ + public static class JoinPredicateInfo { + private final ImmutableList nonEquiJoinPredicateElements; + private final ImmutableList equiJoinPredicateElements; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; + private final ImmutableMap> mapOfProjIndxInJoinSchemaToLeafPInfo; + + public JoinPredicateInfo(List nonEquiJoinPredicateElements, + List equiJoinPredicateElements, + Set projsFromLeftPartOfJoinKeysInChildSchema, + Set projsFromRightPartOfJoinKeysInChildSchema, + Set projsFromRightPartOfJoinKeysInJoinSchema, + Map> mapOfProjIndxInJoinSchemaToLeafPInfo) { + this.nonEquiJoinPredicateElements = ImmutableList.copyOf(nonEquiJoinPredicateElements); + this.equiJoinPredicateElements = ImmutableList.copyOf(equiJoinPredicateElements); + this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); + this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + .copyOf(projsFromRightPartOfJoinKeysInChildSchema); + this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); + this.mapOfProjIndxInJoinSchemaToLeafPInfo = ImmutableMap + .copyOf(mapOfProjIndxInJoinSchemaToLeafPInfo); + } + + public List getNonEquiJoinPredicateElements() { + return nonEquiJoinPredicateElements; + } + + public List getEquiJoinPredicateElements() { + return equiJoinPredicateElements; + } + + public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { + return projsFromLeftPartOfJoinKeysInChildSchema; + } + + public Set getProjsFromRightPartOfJoinKeysInChildSchema() { + return projsFromRightPartOfJoinKeysInChildSchema; + } + + /** + * NOTE: Join Schema = left Schema + (right Schema offset by + * left.fieldcount). Hence its ok to return projections from left in child + * schema. + */ + public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { + return projsFromLeftPartOfJoinKeysInChildSchema; + } + + public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { + return projsFromRightPartOfJoinKeysInJoinSchema; + } + + public Map> getMapOfProjIndxToLeafPInfo() { + return mapOfProjIndxInJoinSchemaToLeafPInfo; + } + + public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { + return constructJoinPredicateInfo(j, j.getCondition()); + } + + public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNode predicate) { + JoinPredicateInfo jpi = null; + JoinLeafPredicateInfo jlpi = null; + List equiLPIList = new ArrayList(); + List nonEquiLPIList = new ArrayList(); + Set projsFromLeftPartOfJoinKeys = new HashSet(); + Set projsFromRightPartOfJoinKeys = new HashSet(); + Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); + Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); + Map> mapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); + List tmpJLPILst = null; + int rightOffSet = j.getLeft().getRowType().getFieldCount(); + int projIndxInJoin; + List conjuctiveElements; + + // 1. Decompose Join condition to a number of leaf predicates + // (conjuctive elements) + conjuctiveElements = RelOptUtil.conjunctions(predicate); + + // 2. Walk through leaf predicates building up JoinLeafPredicateInfo + for (RexNode ce : conjuctiveElements) { + // 2.1 Construct JoinLeafPredicateInfo + jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); + + // 2.2 Classify leaf predicate as Equi vs Non Equi + if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { + equiLPIList.add(jlpi); + } else { + nonEquiLPIList.add(jlpi); + } + + // 2.3 Maintain join keys coming from left vs right (in child & + // Join Schema) + projsFromLeftPartOfJoinKeys.addAll(jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()); + projsFromRightPartOfJoinKeys.addAll(jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()); + projsFromRightPartOfJoinKeysInJoinSchema.addAll(jlpi + .getProjsFromRightPartOfJoinKeysInJoinSchema()); + + // 2.4 Update Join Key to JoinLeafPredicateInfo map with keys + // from left + for (Integer projIndx : jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) { + tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndx); + if (tmpJLPILst == null) + tmpJLPILst = new ArrayList(); + tmpJLPILst.add(jlpi); + tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst); + } + + // 2.5 Update Join Key to JoinLeafPredicateInfo map with keys + // from right + for (Integer projIndx : jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()) { + projIndxInJoin = projIndx + rightOffSet; + tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndxInJoin); + if (tmpJLPILst == null) + tmpJLPILst = new ArrayList(); + tmpJLPILst.add(jlpi); + tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndxInJoin, tmpJLPILst); + } + + } + + // 3. Update Update Join Key to List to use + // ImmutableList + for (Entry> e : tmpMapOfProjIndxInJoinSchemaToLeafPInfo + .entrySet()) { + mapOfProjIndxInJoinSchemaToLeafPInfo.put(e.getKey(), ImmutableList.copyOf(e.getValue())); + } + + // 4. Construct JoinPredicateInfo + jpi = new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, + projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, + mapOfProjIndxInJoinSchemaToLeafPInfo); + return jpi; + } + } + + /** + * JoinLeafPredicateInfo represents leaf predicate in Join condition + * (conjuctive lement).
+ *

+ * JoinLeafPredicateInfo:
+ * 1. Stores list of expressions from left and right child which is part of + * equi join keys.
+ * 2. Stores set of projection indexes from left and right child which is part + * of equi join keys; the indexes are both in child and Join node schema.
+ */ + public static class JoinLeafPredicateInfo { + private final SqlKind m_comparisonType; + private final ImmutableList m_joinKeyExprsFromLeft; + private final ImmutableList m_joinKeyExprsFromRight; + private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; + + public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, + List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, + Set projsFromRightPartOfJoinKeysInChildSchema, + Set projsFromRightPartOfJoinKeysInJoinSchema) { + m_comparisonType = comparisonType; + m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); + m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); + m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); + m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + .copyOf(projsFromRightPartOfJoinKeysInChildSchema); + m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); + } + + public List getJoinKeyExprsFromLeft() { + return m_joinKeyExprsFromLeft; + } + + public List getJoinKeyExprsFromRight() { + return m_joinKeyExprsFromRight; + } + + public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { + return m_projsFromLeftPartOfJoinKeysInChildSchema; + } + + /** + * NOTE: Join Schema = left Schema + (right Schema offset by + * left.fieldcount). Hence its ok to return projections from left in child + * schema. + */ + public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { + return m_projsFromLeftPartOfJoinKeysInChildSchema; + } + + public Set getProjsFromRightPartOfJoinKeysInChildSchema() { + return m_projsFromRightPartOfJoinKeysInChildSchema; + } + + public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { + return m_projsFromRightPartOfJoinKeysInJoinSchema; + } + + private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { + JoinLeafPredicateInfo jlpi = null; + List filterNulls = new ArrayList(); + List joinKeyExprsFromLeft = new ArrayList(); + List joinKeyExprsFromRight = new ArrayList(); + Set projsFromLeftPartOfJoinKeysInChildSchema = new HashSet(); + Set projsFromRightPartOfJoinKeysInChildSchema = new HashSet(); + Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); + int rightOffSet = j.getLeft().getRowType().getFieldCount(); + + // 1. Split leaf join predicate to expressions from left, right + @SuppressWarnings("unused") + RexNode nonEquiPredicate = RelOptUtil.splitJoinCondition(j.getSystemFieldList(), j.getLeft(), + j.getRight(), pe, joinKeyExprsFromLeft, joinKeyExprsFromRight, filterNulls, null); + + // 2. For left expressions, collect child projection indexes used + InputReferencedVisitor irvLeft = new InputReferencedVisitor(); + irvLeft.apply(joinKeyExprsFromLeft); + projsFromLeftPartOfJoinKeysInChildSchema.addAll(irvLeft.inputPosReferenced); + + // 3. For right expressions, collect child projection indexes used + InputReferencedVisitor irvRight = new InputReferencedVisitor(); + irvRight.apply(joinKeyExprsFromRight); + projsFromRightPartOfJoinKeysInChildSchema.addAll(irvRight.inputPosReferenced); + + // 3. Translate projection indexes from right to join schema, by adding + // offset. + for (Integer indx : projsFromRightPartOfJoinKeysInChildSchema) { + projsFromRightPartOfJoinKeysInJoinSchema.add(indx + rightOffSet); + } + + // 4. Construct JoinLeafPredicateInfo + jlpi = new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, + projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, + projsFromRightPartOfJoinKeysInJoinSchema); + + return jlpi; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/JoinUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/JoinUtil.java deleted file mode 100644 index da77d36..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/JoinUtil.java +++ /dev/null @@ -1,295 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.eigenbase.relopt.RelOptUtil; -import org.eigenbase.relopt.RelOptUtil.InputReferencedVisitor; -import org.eigenbase.rex.RexNode; -import org.eigenbase.sql.SqlKind; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; - -/** - * Utility for inspecting Join Conditions.
- *

- * Main Elements:
- * 1. JoinPredicateInfo - represents Join Condition.
- * 2. JoinLeafPredicateInfo - represents leaf predicates with in join condition. - * - * TODO: Move this to Optiq Framework - */ -public class JoinUtil { - - /** - * JoinPredicateInfo represents Join condition; JoinPredicate Info uses - * JoinLeafPredicateInfo to represent individual conjunctive elements in the - * predicate.
- * JoinPredicateInfo = JoinLeafPredicateInfo1 and JoinLeafPredicateInfo2...
- *

- * JoinPredicateInfo:
- * 1. preserves the order of conjuctive elements for - * equi-join(m_equiJoinPredicateElements)
- * 2. Stores set of projection indexes from left and right child which is part - * of equi join keys; the indexes are both in child and Join node schema.
- * 3. Keeps a map of projection indexes that are part of join keys to list of - * conjuctive elements(JoinLeafPredicateInfo) that uses them. - * - */ - public static class JoinPredicateInfo { - private final ImmutableList m_nonEquiJoinPredicateElements; - private final ImmutableList m_equiJoinPredicateElements; - private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; - private final ImmutableMap> m_mapOfProjIndxInJoinSchemaToLeafPInfo; - - public JoinPredicateInfo(List nonEquiJoinPredicateElements, - List equiJoinPredicateElements, - Set projsFromLeftPartOfJoinKeysInChildSchema, - Set projsFromRightPartOfJoinKeysInChildSchema, - Set projsFromRightPartOfJoinKeysInJoinSchema, - Map> mapOfProjIndxInJoinSchemaToLeafPInfo) { - m_nonEquiJoinPredicateElements = ImmutableList.copyOf(nonEquiJoinPredicateElements); - m_equiJoinPredicateElements = ImmutableList.copyOf(equiJoinPredicateElements); - m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet - .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet - .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet - .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); - m_mapOfProjIndxInJoinSchemaToLeafPInfo = ImmutableMap - .copyOf(mapOfProjIndxInJoinSchemaToLeafPInfo); - } - - public List getNonEquiJoinPredicateElements() { - return m_nonEquiJoinPredicateElements; - } - - public List getEquiJoinPredicateElements() { - return m_equiJoinPredicateElements; - } - - public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; - } - - public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return m_projsFromRightPartOfJoinKeysInChildSchema; - } - - /** - * NOTE: Join Schema = left Schema + (right Schema offset by - * left.fieldcount). Hence its ok to return projections from left in child - * schema. - */ - public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; - } - - public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return m_projsFromRightPartOfJoinKeysInJoinSchema; - } - - public Map> getMapOfProjIndxToLeafPInfo() { - return m_mapOfProjIndxInJoinSchemaToLeafPInfo; - } - - public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { - return constructJoinPredicateInfo(j, j.getCondition()); - } - - public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNode predicate) { - JoinPredicateInfo jpi = null; - JoinLeafPredicateInfo jlpi = null; - List equiLPIList = new ArrayList(); - List nonEquiLPIList = new ArrayList(); - Set projsFromLeftPartOfJoinKeys = new HashSet(); - Set projsFromRightPartOfJoinKeys = new HashSet(); - Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); - Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); - Map> mapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); - List tmpJLPILst = null; - int rightOffSet = j.getLeft().getRowType().getFieldCount(); - int projIndxInJoin; - List conjuctiveElements; - - todo("Move this to Optiq"); - - // 1. Decompose Join condition to a number of leaf predicates - // (conjuctive elements) - conjuctiveElements = RelOptUtil.conjunctions(predicate); - - // 2. Walk through leaf predicates building up JoinLeafPredicateInfo - for (RexNode ce : conjuctiveElements) { - // 2.1 Construct JoinLeafPredicateInfo - jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); - - // 2.2 Classify leaf predicate as Equi vs Non Equi - if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { - equiLPIList.add(jlpi); - } else { - nonEquiLPIList.add(jlpi); - } - - // 2.3 Maintain join keys coming from left vs right (in child & - // Join Schema) - projsFromLeftPartOfJoinKeys.addAll(jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()); - projsFromRightPartOfJoinKeys.addAll(jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()); - projsFromRightPartOfJoinKeysInJoinSchema.addAll(jlpi - .getProjsFromRightPartOfJoinKeysInJoinSchema()); - - // 2.4 Update Join Key to JoinLeafPredicateInfo map with keys - // from left - for (Integer projIndx : jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) { - tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndx); - if (tmpJLPILst == null) - tmpJLPILst = new ArrayList(); - tmpJLPILst.add(jlpi); - tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst); - } - - // 2.5 Update Join Key to JoinLeafPredicateInfo map with keys - // from right - for (Integer projIndx : jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()) { - projIndxInJoin = projIndx + rightOffSet; - tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndxInJoin); - if (tmpJLPILst == null) - tmpJLPILst = new ArrayList(); - tmpJLPILst.add(jlpi); - tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndxInJoin, tmpJLPILst); - } - - } - - // 3. Update Update Join Key to List to use - // ImmutableList - for (Entry> e : tmpMapOfProjIndxInJoinSchemaToLeafPInfo - .entrySet()) { - mapOfProjIndxInJoinSchemaToLeafPInfo.put(e.getKey(), ImmutableList.copyOf(e.getValue())); - } - - // 4. Construct JoinPredicateInfo - jpi = new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, - projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, - mapOfProjIndxInJoinSchemaToLeafPInfo); - return jpi; - } - } - - /** - * JoinLeafPredicateInfo represents leaf predicate in Join condition - * (conjuctive lement).
- *

- * JoinLeafPredicateInfo:
- * 1. Stores list of expressions from left and right child which is part of - * equi join keys.
- * 2. Stores set of projection indexes from left and right child which is part - * of equi join keys; the indexes are both in child and Join node schema.
- */ - public static class JoinLeafPredicateInfo { - private final SqlKind m_comparisonType; - private final ImmutableList m_joinKeyExprsFromLeft; - private final ImmutableList m_joinKeyExprsFromRight; - private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; - - public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, - List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, - Set projsFromRightPartOfJoinKeysInChildSchema, - Set projsFromRightPartOfJoinKeysInJoinSchema) { - m_comparisonType = comparisonType; - m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); - m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); - m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet - .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet - .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet - .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); - } - - public List getJoinKeyExprsFromLeft() { - return m_joinKeyExprsFromLeft; - } - - public List getJoinKeyExprsFromRight() { - return m_joinKeyExprsFromRight; - } - - public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; - } - - /** - * NOTE: Join Schema = left Schema + (right Schema offset by - * left.fieldcount). Hence its ok to return projections from left in child - * schema. - */ - public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; - } - - public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return m_projsFromRightPartOfJoinKeysInChildSchema; - } - - public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return m_projsFromRightPartOfJoinKeysInJoinSchema; - } - - public static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { - JoinLeafPredicateInfo jlpi = null; - List filterNulls = new ArrayList(); - List joinKeyExprsFromLeft = new ArrayList(); - List joinKeyExprsFromRight = new ArrayList(); - Set projsFromLeftPartOfJoinKeysInChildSchema = new HashSet(); - Set projsFromRightPartOfJoinKeysInChildSchema = new HashSet(); - Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); - int rightOffSet = j.getLeft().getRowType().getFieldCount(); - - todo("Move this to Optiq"); - - // 1. Split leaf join predicate to expressions from left, right - @SuppressWarnings("unused") - RexNode nonEquiPredicate = RelOptUtil.splitJoinCondition(j.getSystemFieldList(), j.getLeft(), - j.getRight(), pe, joinKeyExprsFromLeft, joinKeyExprsFromRight, filterNulls, null); - - // 2. For left expressions, collect child projection indexes used - InputReferencedVisitor irvLeft = new InputReferencedVisitor(); - irvLeft.apply(joinKeyExprsFromLeft); - projsFromLeftPartOfJoinKeysInChildSchema.addAll(irvLeft.inputPosReferenced); - - // 3. For right expressions, collect child projection indexes used - InputReferencedVisitor irvRight = new InputReferencedVisitor(); - irvRight.apply(joinKeyExprsFromRight); - projsFromRightPartOfJoinKeysInChildSchema.addAll(irvRight.inputPosReferenced); - - // 3. Translate projection indexes from right to join schema, by adding - // offset. - for (Integer indx : projsFromRightPartOfJoinKeysInChildSchema) { - projsFromRightPartOfJoinKeysInJoinSchema.add(indx + rightOffSet); - } - - // 4. Construct JoinLeafPredicateInfo - jlpi = new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, - projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, - projsFromRightPartOfJoinKeysInJoinSchema); - - return jlpi; - } - } - - @Deprecated - public static void todo(String s) { - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/Pair.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/Pair.java deleted file mode 100644 index c923340..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/Pair.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq; - -public class Pair { - private final T1 m_first; - private final T2 m_second; - - public Pair(T1 first, T2 second) { - m_first = first; - m_second = second; - } - - public T1 getFirst() { - return m_first; - } - - public T2 getSecond() { - return m_second; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java index d3e517d..c3706dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq; import java.util.ArrayList; @@ -7,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -38,16 +54,15 @@ import com.google.common.collect.ImmutableMap.Builder; public class RelOptHiveTable extends RelOptAbstractTable { - private final Table m_hiveTblMetadata; - private final ImmutableList m_hiveNonPartitionCols; - private final ImmutableMap m_hiveNonPartitionColsMap; - private final ImmutableMap m_hivePartitionColsMap; - private final int m_noOfProjs; - final HiveConf m_hiveConf; - - private double m_rowCount = -1; - Map m_hiveColStatsMap = new HashMap(); - private Integer m_numPartitions; + private final Table hiveTblMetadata; + private final ImmutableList hiveNonPartitionCols; + private final ImmutableMap hiveNonPartitionColsMap; + private final ImmutableMap hivePartitionColsMap; + private final int noOfProjs; + final HiveConf hiveConf; + + private double rowCount = -1; + Map hiveColStatsMap = new HashMap(); PrunedPartitionList partitionList; Map partitionCache; AtomicInteger noColsMissingStats; @@ -60,12 +75,12 @@ public RelOptHiveTable(RelOptSchema optiqSchema, String name, RelDataType rowTyp Table hiveTblMetadata, List hiveNonPartitionCols, List hivePartitionCols, HiveConf hconf, Map partitionCache, AtomicInteger noColsMissingStats) { super(optiqSchema, name, rowType); - m_hiveTblMetadata = hiveTblMetadata; - m_hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols); - m_hiveNonPartitionColsMap = getColInfoMap(hiveNonPartitionCols, 0); - m_hivePartitionColsMap = getColInfoMap(hivePartitionCols, m_hiveNonPartitionColsMap.size()); - m_noOfProjs = hiveNonPartitionCols.size() + hivePartitionCols.size(); - m_hiveConf = hconf; + this.hiveTblMetadata = hiveTblMetadata; + this.hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols); + this.hiveNonPartitionColsMap = getColInfoMap(hiveNonPartitionCols, 0); + this.hivePartitionColsMap = getColInfoMap(hivePartitionCols, hiveNonPartitionColsMap.size()); + this.noOfProjs = hiveNonPartitionCols.size() + hivePartitionCols.size(); + this.hiveConf = hconf; this.partitionCache = partitionCache; this.noColsMissingStats = noColsMissingStats; } @@ -100,27 +115,27 @@ public RelNode toRel(ToRelContext context) { @Override public double getRowCount() { - if (m_rowCount == -1) { + if (rowCount == -1) { if (null == partitionList) { // we are here either unpartitioned table or partitioned table with no predicates - computePartitionList(m_hiveConf, null); + computePartitionList(hiveConf, null); } - if (m_hiveTblMetadata.isPartitioned()) { + if (hiveTblMetadata.isPartitioned()) { List rowCounts = StatsUtils.getBasicStatForPartitions( - m_hiveTblMetadata, partitionList.getNotDeniedPartns(), + hiveTblMetadata, partitionList.getNotDeniedPartns(), StatsSetupConst.ROW_COUNT); - m_rowCount = StatsUtils.getSumIgnoreNegatives(rowCounts); + rowCount = StatsUtils.getSumIgnoreNegatives(rowCounts); } else { - m_rowCount = StatsUtils.getNumRows(m_hiveTblMetadata); + rowCount = StatsUtils.getNumRows(hiveTblMetadata); } } - return m_rowCount; + return rowCount; } public Table getHiveTableMD() { - return m_hiveTblMetadata; + return hiveTblMetadata; } private String getColNamesForLogging(Set colLst) { @@ -140,16 +155,16 @@ private String getColNamesForLogging(Set colLst) { public void computePartitionList(HiveConf conf, RexNode pruneNode) { try { - if (!m_hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0 ) { + if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0 ) { // there is no predicate on partitioning column, we need all partitions in this case. - partitionList = PartitionPruner.prune(m_hiveTblMetadata, null, conf, getName(), partitionCache); + partitionList = PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); return; } // We have valid pruning expressions, only retrieve qualifying partitions ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true)); - partitionList = PartitionPruner.prune(m_hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); + partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); } catch (HiveException he) { throw new RuntimeException(he); } @@ -165,16 +180,16 @@ private void updateColStats(Set projIndxLst) { // 1. Separate required columns to Non Partition and Partition Cols ColumnInfo tmp; for (Integer pi : projIndxLst) { - if (m_hiveColStatsMap.get(pi) == null) { - if ((tmp = m_hiveNonPartitionColsMap.get(pi)) != null) { + if (hiveColStatsMap.get(pi) == null) { + if ((tmp = hiveNonPartitionColsMap.get(pi)) != null) { nonPartColNamesThatRqrStats.add(tmp.getInternalName()); nonPartColIndxsThatRqrStats.add(pi); - } else if ((tmp = m_hivePartitionColsMap.get(pi)) != null) { + } else if ((tmp = hivePartitionColsMap.get(pi)) != null) { partColNamesThatRqrStats.add(tmp.getInternalName()); partColIndxsThatRqrStats.add(pi); } else { String logMsg = "Unable to find Column Index: " + pi + ", in " - + m_hiveTblMetadata.getCompleteName(); + + hiveTblMetadata.getCompleteName(); LOG.error(logMsg); throw new RuntimeException(logMsg); } @@ -184,16 +199,16 @@ private void updateColStats(Set projIndxLst) { if (null == partitionList) { // We could be here either because its an unpartitioned table or because // there are no pruning predicates on a partitioned table. - computePartitionList(m_hiveConf, null); + computePartitionList(hiveConf, null); } // 2. Obtain Col Stats for Non Partition Cols if (nonPartColNamesThatRqrStats.size() > 0) { List hiveColStats; - if (!m_hiveTblMetadata.isPartitioned()) { + if (!hiveTblMetadata.isPartitioned()) { // 2.1 Handle the case for unpartitioned table. - hiveColStats = StatsUtils.getTableColumnStats(m_hiveTblMetadata, m_hiveNonPartitionCols, + hiveColStats = StatsUtils.getTableColumnStats(hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats); // 2.1.1 Record Column Names that we needed stats for but couldn't @@ -215,17 +230,17 @@ private void updateColStats(Set projIndxLst) { try { if (partitionList.getNotDeniedPartns().isEmpty()) { // no need to make a metastore call - m_rowCount = 0; + rowCount = 0; hiveColStats = new ArrayList(); for (String c : nonPartColNamesThatRqrStats) { // add empty stats object for each column - hiveColStats.add(new ColStatistics(m_hiveTblMetadata.getTableName(), c, null)); + hiveColStats.add(new ColStatistics(hiveTblMetadata.getTableName(), c, null)); } colNamesFailedStats.clear(); } else { - Statistics stats = StatsUtils.collectStatistics(m_hiveConf, partitionList, - m_hiveTblMetadata, m_hiveNonPartitionCols, nonPartColNamesThatRqrStats, true, true); - m_rowCount = stats.getNumRows(); + Statistics stats = StatsUtils.collectStatistics(hiveConf, partitionList, + hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats, true, true); + rowCount = stats.getNumRows(); hiveColStats = new ArrayList(); for (String c : nonPartColNamesThatRqrStats) { ColStatistics cs = stats.getColumnStatisticsFromColName(c); @@ -245,27 +260,26 @@ private void updateColStats(Set projIndxLst) { if (hiveColStats != null && hiveColStats.size() == nonPartColNamesThatRqrStats.size()) { for (int i = 0; i < hiveColStats.size(); i++) { - m_hiveColStatsMap.put(nonPartColIndxsThatRqrStats.get(i), hiveColStats.get(i)); + hiveColStatsMap.put(nonPartColIndxsThatRqrStats.get(i), hiveColStats.get(i)); } } } // 3. Obtain Stats for Partition Cols if (colNamesFailedStats.isEmpty() && !partColNamesThatRqrStats.isEmpty()) { - m_numPartitions = partitionList.getPartitions().size(); ColStatistics cStats = null; for (int i = 0; i < partColNamesThatRqrStats.size(); i++) { - cStats = new ColStatistics(m_hiveTblMetadata.getTableName(), - partColNamesThatRqrStats.get(i), m_hivePartitionColsMap.get( + cStats = new ColStatistics(hiveTblMetadata.getTableName(), + partColNamesThatRqrStats.get(i), hivePartitionColsMap.get( partColIndxsThatRqrStats.get(i)).getTypeName()); cStats.setCountDistint(getDistinctCount(partitionList.getPartitions(),partColNamesThatRqrStats.get(i))); - m_hiveColStatsMap.put(partColIndxsThatRqrStats.get(i), cStats); + hiveColStatsMap.put(partColIndxsThatRqrStats.get(i), cStats); } } // 4. Warn user if we could get stats for required columns if (!colNamesFailedStats.isEmpty()) { - String logMsg = "No Stats for " + m_hiveTblMetadata.getCompleteName() + ", Columns: " + String logMsg = "No Stats for " + hiveTblMetadata.getCompleteName() + ", Columns: " + getColNamesForLogging(colNamesFailedStats); LOG.error(logMsg); noColsMissingStats.getAndAdd(colNamesFailedStats.size()); @@ -287,16 +301,16 @@ private int getDistinctCount(Set partitions, String partColName) { if (projIndxLst != null) { updateColStats(new HashSet(projIndxLst)); for (Integer i : projIndxLst) { - colStatsBldr.add(m_hiveColStatsMap.get(i)); + colStatsBldr.add(hiveColStatsMap.get(i)); } } else { List pILst = new ArrayList(); - for (Integer i = 0; i < m_noOfProjs; i++) { + for (Integer i = 0; i < noOfProjs; i++) { pILst.add(i); } updateColStats(new HashSet(pILst)); for (Integer pi : pILst) { - colStatsBldr.add(m_hiveColStatsMap.get(pi)); + colStatsBldr.add(hiveColStatsMap.get(pi)); } } @@ -312,7 +326,7 @@ private int getDistinctCount(Set partitions, String partColName) { public boolean containsPartitionColumnsOnly(BitSet cols) { for (int i = cols.nextSetBit(0); i >= 0; i++, i = cols.nextSetBit(i + 1)) { - if (!m_hivePartitionColsMap.containsKey(i)) { + if (!hivePartitionColsMap.containsKey(i)) { return false; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java index d4bd678..9b653d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java @@ -1,52 +1,19 @@ package org.apache.hadoop.hive.ql.optimizer.optiq; -import java.util.List; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; -import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.RelCollation; import org.eigenbase.rel.RelCollationImpl; -import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; -import org.eigenbase.reltype.RelDataType; public class TraitsUtil { - - public static RelTraitSet getSelectTraitSet(RelOptCluster cluster, RelNode child) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - public static RelTraitSet getSortTraitSet(RelOptCluster cluster, RelTraitSet traitSet, RelCollation collation) { return traitSet.plus(collation); } - public static RelTraitSet getFilterTraitSet(RelOptCluster cluster, RelTraitSet traitSet, - RelNode child) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - - public static RelTraitSet getLimitTraitSet(RelOptCluster cluster, RelTraitSet traitSet, - RelNode child) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - - public static RelTraitSet getAggregateTraitSet(RelOptCluster cluster, RelTraitSet traitSet, - List gbCols, List aggCalls, RelNode child) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - - public static RelTraitSet getTableScanTraitSet(RelOptCluster cluster, RelTraitSet traitSet, - RelOptHiveTable table, RelDataType rowtype) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - - public static RelTraitSet getJoinTraitSet(RelOptCluster cluster, RelTraitSet traitSet) { - return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - } - - public static RelTraitSet getUnionTraitSet(RelOptCluster cluster, RelTraitSet traitSet) { + public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) { return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java index 34a37e4..a4df413 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.cost; import org.eigenbase.relopt.RelOptCost; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java index 926bca5..257c380 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.cost; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; @@ -10,6 +27,7 @@ private static final double localFSWriteCostInNanoSec = 4 * netCostInNanoSec; private static final double localFSReadCostInNanoSec = 4 * netCostInNanoSec; private static final double hDFSWriteCostInNanoSec = 10 * localFSWriteCostInNanoSec; + @SuppressWarnings("unused") private static final double hDFSReadCostInNanoSec = 1.5 * localFSReadCostInNanoSec; public static RelOptCost computCardinalityBasedCost(HiveRel hr) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java index c81fd8a..fc19895 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java @@ -1,10 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.BitSet; import java.util.List; -import net.hydromatic.optiq.util.BitSets; - import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveCost; import org.eigenbase.rel.AggregateCall; @@ -24,8 +39,7 @@ public HiveAggregateRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, BitSet groupSet, List aggCalls) throws InvalidRelException { - super(cluster, TraitsUtil.getAggregateTraitSet(cluster, traitSet, BitSets.toList(groupSet), - aggCalls, child), child, groupSet, aggCalls); + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, groupSet, aggCalls); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java index ebf420d..8b85046 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java @@ -1,7 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; -import java.util.List; - import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveCost; import org.eigenbase.rel.FilterRelBase; @@ -18,7 +33,7 @@ public static final FilterFactory DEFAULT_FILTER_FACTORY = new HiveFilterFactoryImpl(); public HiveFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { - super(cluster, TraitsUtil.getFilterTraitSet(cluster, traits, child), child, condition); + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); } @Override @@ -45,8 +60,7 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { @Override public RelNode createFilter(RelNode child, RexNode condition) { RelOptCluster cluster = child.getCluster(); - HiveFilterRel filter = new HiveFilterRel(cluster, TraitsUtil.getFilterTraitSet(cluster, null, - child), child, condition); + HiveFilterRel filter = new HiveFilterRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); return filter; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java index 6f642b2..363e9ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.Collections; @@ -36,9 +53,10 @@ public static final JoinFactory HIVE_JOIN_FACTORY = new HiveJoinFactoryImpl(); - private final boolean m_leftSemiJoin; - private final JoinAlgorithm m_joinAlgorithm; - private final MapJoinStreamingRelation m_mapJoinStreamingSide = MapJoinStreamingRelation.NONE; + private final boolean leftSemiJoin; + private final JoinAlgorithm joinAlgorithm; + @SuppressWarnings("unused") + private final MapJoinStreamingRelation mapJoinStreamingSide = MapJoinStreamingRelation.NONE; public static HiveJoinRel getJoin(RelOptCluster cluster, RelNode left, RelNode right, RexNode condition, JoinRelType joinType, boolean leftSemiJoin) { @@ -55,10 +73,10 @@ protected HiveJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, R RexNode condition, JoinRelType joinType, Set variablesStopped, JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin, boolean leftSemiJoin) throws InvalidRelException { - super(cluster, TraitsUtil.getJoinTraitSet(cluster, traits), left, right, condition, joinType, + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), left, right, condition, joinType, variablesStopped); - this.m_joinAlgorithm = joinAlgo; - m_leftSemiJoin = leftSemiJoin; + this.joinAlgorithm = joinAlgo; + this.leftSemiJoin = leftSemiJoin; } @Override @@ -71,7 +89,7 @@ public final HiveJoinRel copy(RelTraitSet traitSet, RexNode conditionExpr, RelNo try { Set variablesStopped = Collections.emptySet(); return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType, - variablesStopped, JoinAlgorithm.NONE, null, m_leftSemiJoin); + variablesStopped, JoinAlgorithm.NONE, null, leftSemiJoin); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. @@ -80,11 +98,11 @@ public final HiveJoinRel copy(RelTraitSet traitSet, RexNode conditionExpr, RelNo } public JoinAlgorithm getJoinAlgorithm() { - return m_joinAlgorithm; + return joinAlgorithm; } public boolean isLeftSemiJoin() { - return m_leftSemiJoin; + return leftSemiJoin; } /** @@ -102,7 +120,7 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { */ @Override public RelDataType deriveRowType() { - if (m_leftSemiJoin) { + if (leftSemiJoin) { return deriveJoinRowType(left.getRowType(), null, JoinRelType.INNER, getCluster().getTypeFactory(), null, Collections. emptyList()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java index bf37a7b..f8755d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.List; @@ -18,7 +35,7 @@ HiveLimitRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RexNode offset, RexNode fetch) { - super(cluster, TraitsUtil.getLimitTraitSet(cluster, traitSet, child), child); + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child); this.offset = offset; this.fetch = fetch; assert getConvention() instanceof HiveRel; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java index a60af2e..68e37f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.ArrayList; @@ -30,7 +47,7 @@ public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl(); - private final List m_virtualCols; + private final List virtualCols; /** * Creates a HiveProjectRel. @@ -49,7 +66,7 @@ public HiveProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, List exps, RelDataType rowType, int flags) { super(cluster, traitSet, child, exps, rowType, flags); - m_virtualCols = ImmutableList.copyOf(HiveOptiqUtil.getVirtualCols(exps)); + virtualCols = ImmutableList.copyOf(HiveOptiqUtil.getVirtualCols(exps)); } /** @@ -73,7 +90,7 @@ public static HiveProjectRel create(RelNode child, List exps, */ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, RelDataType rowType, final List collationList) { - RelTraitSet traitSet = TraitsUtil.getSelectTraitSet(cluster, child); + RelTraitSet traitSet = TraitsUtil.getDefaultTraitSet(cluster); return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -145,7 +162,7 @@ public void implement(Implementor implementor) { } public List getVirtualCols() { - return m_virtualCols; + return virtualCols; } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java index 6f3f1d8..4738c4a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import org.eigenbase.rel.RelNode; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java index 3bd7889..dc8f614 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java index 4fe1735..bd66459 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.List; @@ -24,7 +41,6 @@ *

*/ public class HiveTableScanRel extends TableAccessRelBase implements HiveRel { - private List m_hiveColStat; /** * Creates a HiveTableScan. @@ -40,7 +56,7 @@ */ public HiveTableScanRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptHiveTable table, RelDataType rowtype) { - super(cluster, TraitsUtil.getTableScanTraitSet(cluster, traitSet, table, rowtype), table); + super(cluster, TraitsUtil.getDefaultTraitSet(cluster), table); assert getConvention() == HiveRel.CONVENTION; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java index ccd52b0..d34fe95 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; import java.util.List; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveMergeProjectRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveMergeProjectRule.java index a34b532..416343a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveMergeProjectRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveMergeProjectRule.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.rules; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePullUpProjectsAboveJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePullUpProjectsAboveJoinRule.java deleted file mode 100644 index a48112e..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePullUpProjectsAboveJoinRule.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.rules; - -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; -import org.eigenbase.rel.ProjectRelBase; -import org.eigenbase.rel.RelNode; -import org.eigenbase.rel.rules.PullUpProjectsAboveJoinRule; -import org.eigenbase.relopt.RelOptRuleOperand; - -public class HivePullUpProjectsAboveJoinRule extends PullUpProjectsAboveJoinRule { - - public static final HivePullUpProjectsAboveJoinRule BOTH_PROJECT = new HivePullUpProjectsAboveJoinRule( - operand( - HiveJoinRel.class, - operand( - ProjectRelBase.class, - any()), - operand( - ProjectRelBase.class, - any())), - "HivePullUpProjectsAboveJoinRule: with two HiveProjectRel children"); - - public static final HivePullUpProjectsAboveJoinRule LEFT_PROJECT = new HivePullUpProjectsAboveJoinRule( - operand( - HiveJoinRel.class, - some(operand( - ProjectRelBase.class, - any()))), - "HivePullUpProjectsAboveJoinRule: with HiveProjectRel on left"); - - public static final HivePullUpProjectsAboveJoinRule RIGHT_PROJECT = new HivePullUpProjectsAboveJoinRule( - operand( - HiveJoinRel.class, - operand(RelNode.class, - any()), - operand( - ProjectRelBase.class, - any())), - "HivePullUpProjectsAboveJoinRule: with HiveProjectRel on right"); - - public HivePullUpProjectsAboveJoinRule(RelOptRuleOperand operand, String description) { - super(operand, description, HiveProjectRel.DEFAULT_PROJECT_FACTORY); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java index f8d1ac1..da0f7a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.rules; import java.util.ArrayList; @@ -24,8 +41,6 @@ import org.eigenbase.sql.SqlKind; import org.eigenbase.util.Holder; -import com.google.common.collect.ImmutableList; - public abstract class HivePushFilterPastJoinRule extends RelOptRule { public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterPastJoinRule( @@ -96,7 +111,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, } final List aboveFilters = filter != null ? RelOptUtil - .conjunctions(filter.getCondition()) : ImmutableList. of(); + .conjunctions(filter.getCondition()) : new ArrayList(); List leftFilters = new ArrayList(); List rightFilters = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java deleted file mode 100644 index 0714bed..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.rules; - -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel.JoinAlgorithm; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; -import org.eigenbase.rel.JoinRelBase; -import org.eigenbase.rel.rules.PushJoinThroughJoinRule; -import org.eigenbase.relopt.RelOptRule; -import org.eigenbase.relopt.RelOptRuleCall; - -public class HivePushJoinThroughJoinRule extends PushJoinThroughJoinRule { - public static final RelOptRule RIGHT = new HivePushJoinThroughJoinRule( - "Hive PushJoinThroughJoinRule:right", true, - HiveJoinRel.class); - public static final RelOptRule LEFT = new HivePushJoinThroughJoinRule( - "Hive PushJoinThroughJoinRule:left", false, - HiveJoinRel.class); - - private HivePushJoinThroughJoinRule(String description, boolean right, - Class clazz) { - super(description, right, clazz, HiveProjectRel.DEFAULT_PROJECT_FACTORY); - } - - @Override - public boolean matches(RelOptRuleCall call) { - boolean isAMatch = false; - final HiveJoinRel topJoin = call.rel(0); - final HiveJoinRel bottomJoin = call.rel(1); - - if (!topJoin.isLeftSemiJoin() && topJoin.getJoinAlgorithm() == JoinAlgorithm.NONE - && bottomJoin.getJoinAlgorithm() == JoinAlgorithm.NONE) { - isAMatch = true; - } - - return isAMatch; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java deleted file mode 100644 index 10a9cb8..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.rules; - -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel.JoinAlgorithm; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; - -import org.eigenbase.rel.rules.SwapJoinRule; -import org.eigenbase.relopt.RelOptRuleCall; - -public class HiveSwapJoinRule extends SwapJoinRule { - public static final HiveSwapJoinRule INSTANCE = new HiveSwapJoinRule(); - - private HiveSwapJoinRule() { - super(HiveJoinRel.class, HiveProjectRel.DEFAULT_PROJECT_FACTORY); - } - - @Override - public boolean matches(RelOptRuleCall call) { - if (call. rel(0).isLeftSemiJoin()) - return false; - else - return super.matches(call) - && call. rel(0).getJoinAlgorithm() == JoinAlgorithm.NONE; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/CBOTableStatsValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/CBOTableStatsValidator.java deleted file mode 100644 index 370feaa..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/CBOTableStatsValidator.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.stats; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Map; -import java.util.Stack; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.ForwardWalker; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; - -import com.google.common.collect.ImmutableMap; - -public class CBOTableStatsValidator { - private final CBOValidateStatsContext m_ctx = new CBOValidateStatsContext(); - - public boolean validStats(Operator sinkOp, ParseContext pCtx) { - Map rules = ImmutableMap - . builder() - .put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), - new TableScanProcessor()).build(); - - Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), rules, m_ctx); - GraphWalker fWalker = new ForwardWalker(disp); - - ArrayList topNodes = new ArrayList(); - topNodes.addAll(pCtx.getTopOps().values()); - - try { - fWalker.startWalking(topNodes, null); - } catch (SemanticException e) { - throw new RuntimeException(e); - } - - return (m_ctx.m_tabsWithIncompleteStats.isEmpty()); - } - - public String getIncompleteStatsTabNames() { - StringBuilder sb = new StringBuilder(); - for (String tabName : m_ctx.m_tabsWithIncompleteStats) { - if (sb.length() > 1) - sb.append(", "); - sb.append(tabName); - } - return sb.toString(); - } - - private static NodeProcessor getDefaultProc() { - return new NodeProcessor() { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) { - return null; - // TODO: Shouldn't we throw exception? as this would imply we got an op - // tree with no TS - } - }; - } - - static class TableScanProcessor implements NodeProcessor { - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) { - TableScanOperator tableScanOp = (TableScanOperator) nd; - Statistics stats = tableScanOp.getStatistics(); - int noColsWithStats = (stats != null && stats.getColumnStats() != null) ? stats - .getColumnStats().size() : 0; - if (noColsWithStats != tableScanOp.getNeededColumns().size()) { - ((CBOValidateStatsContext) procCtx).m_tabsWithIncompleteStats.add(tableScanOp.getConf() - .getAlias()); - } - return null; - } - } - - static class CBOValidateStatsContext implements NodeProcessorCtx { - final private HashSet m_tabsWithIncompleteStats = new HashSet(); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java index f8e3238..34e958c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java @@ -5,7 +5,6 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; import org.eigenbase.rel.FilterRelBase; -import org.eigenbase.rel.ProjectRel; import org.eigenbase.rel.ProjectRelBase; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.RelMetadataQuery; @@ -14,7 +13,6 @@ import org.eigenbase.rex.RexCall; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexUtil; import org.eigenbase.rex.RexVisitorImpl; import org.eigenbase.sql.SqlKind; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java index f3e91bb..4be57b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.stats; import java.util.BitSet; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java index 0c3b399..5d9b145 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.stats; import java.util.ArrayList; @@ -8,8 +25,8 @@ import net.hydromatic.optiq.BuiltinMethod; -import org.apache.hadoop.hive.ql.optimizer.optiq.JoinUtil.JoinLeafPredicateInfo; -import org.apache.hadoop.hive.ql.optimizer.optiq.JoinUtil.JoinPredicateInfo; +import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil.JoinLeafPredicateInfo; +import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil.JoinPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; import org.eigenbase.rel.JoinRelType; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java index f9c0aeb..bfa4255 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java @@ -1,10 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; -import java.sql.Date; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; +import net.hydromatic.avatica.ByteString; + import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -117,6 +135,11 @@ static ASTNode literal(RexLiteral literal) { SqlTypeName sqlType = literal.getType().getSqlTypeName(); switch (sqlType) { + case BINARY: + ByteString bs = (ByteString) literal.getValue(); + val = bs.byteAt(0); + type = HiveParser.BigintLiteral; + break; case TINYINT: val = literal.getValue3(); type = HiveParser.TinyintLiteral; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java index 9189e63..0ba320c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; import java.util.ArrayList; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java index dd2bf22..3b204b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; import java.util.ArrayList; @@ -24,13 +41,9 @@ import org.eigenbase.rel.rules.MultiJoinRel; import org.eigenbase.relopt.hep.HepRelVertex; import org.eigenbase.relopt.volcano.RelSubset; -import org.eigenbase.reltype.RelDataTypeField; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; -import com.google.common.base.Function; -import com.google.common.collect.Lists; - public class DerivedTableInjector { public static RelNode convertOpTree(RelNode rel, List resultSchema) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java index 5636919..933b687 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeField; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java deleted file mode 100644 index c051b65..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java +++ /dev/null @@ -1,697 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.translator; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.Stack; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.LimitOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.ForwardWalker; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveAggregateRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveSortRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; -import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AggregationDesc; -import org.apache.hadoop.hive.ql.plan.ColStatistics; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.JoinCondDesc; -import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; -import org.eigenbase.rel.AggregateCall; -import org.eigenbase.rel.Aggregation; -import org.eigenbase.rel.InvalidRelException; -import org.eigenbase.rel.JoinRelType; -import org.eigenbase.rel.RelCollation; -import org.eigenbase.rel.RelCollationImpl; -import org.eigenbase.rel.RelFieldCollation; -import org.eigenbase.rel.RelNode; -import org.eigenbase.rel.TableAccessRelBase; -import org.eigenbase.relopt.RelOptCluster; -import org.eigenbase.relopt.RelOptSchema; -import org.eigenbase.relopt.RelTraitSet; -import org.eigenbase.reltype.RelDataType; -import org.eigenbase.reltype.RelDataTypeField; -import org.eigenbase.rex.RexCall; -import org.eigenbase.rex.RexInputRef; -import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexUtil; -import org.eigenbase.sql.fun.SqlStdOperatorTable; -import org.eigenbase.util.CompositeList; -import org.eigenbase.util.Pair; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; - -public class RelNodeConverter { - private static final Map AGG_MAP = ImmutableMap - . builder() - .put( - "count", - SqlStdOperatorTable.COUNT) - .put("sum", SqlStdOperatorTable.SUM) - .put("min", SqlStdOperatorTable.MIN) - .put("max", SqlStdOperatorTable.MAX) - .put("avg", SqlStdOperatorTable.AVG) - .build(); - - public static RelNode convert(Operator sinkOp, RelOptCluster cluster, - RelOptSchema schema, SemanticAnalyzer sA, ParseContext pCtx) { - - Context ctx = new Context(cluster, schema, sA, pCtx); - - Map rules = ImmutableMap - . builder() - .put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), - new TableScanProcessor()) - .put(new RuleRegExp("R2", FilterOperator.getOperatorName() + "%"), new FilterProcessor()) - .put(new RuleRegExp("R3", SelectOperator.getOperatorName() + "%"), new SelectProcessor()) - .put(new RuleRegExp("R4", JoinOperator.getOperatorName() + "%"), new JoinProcessor()) - .put(new RuleRegExp("R5", LimitOperator.getOperatorName() + "%"), new LimitProcessor()) - .put(new RuleRegExp("R6", GroupByOperator.getOperatorName() + "%"), new GroupByProcessor()) - .put(new RuleRegExp("R7", ReduceSinkOperator.getOperatorName() + "%"), - new ReduceSinkProcessor()).build(); - - Dispatcher disp = new DefaultRuleDispatcher(new DefaultProcessor(), rules, ctx); - GraphWalker egw = new ForwardWalker(disp); - - ArrayList topNodes = new ArrayList(); - topNodes.addAll(pCtx.getTopOps().values()); - - HashMap outputMap = new HashMap(); - try { - egw.startWalking(topNodes, outputMap); - } catch (SemanticException se) { - // @revisit - throw new RuntimeException(se); - } - return (HiveRel) outputMap.get(sinkOp); - } - - static class Context implements NodeProcessorCtx { - RelOptCluster cluster; - RelOptSchema schema; - SemanticAnalyzer sA; - ParseContext parseCtx; - /* - * A Map from hive column internalNames to Optiq positions. A separate map - * for each Operator. - */ - Map> opPositionMap; - - Map, RelNode> hiveOpToRelNode; - - public Context(RelOptCluster cluster, RelOptSchema schema, SemanticAnalyzer sA, - ParseContext parseCtx) { - super(); - this.cluster = cluster; - this.schema = schema; - this.sA = sA; - this.parseCtx = parseCtx; - opPositionMap = new HashMap>(); - hiveOpToRelNode = new HashMap, RelNode>(); - } - - void buildColumnMap(Operator op, RelNode rNode) { - RowSchema rr = op.getSchema(); - ImmutableMap.Builder b = new ImmutableMap.Builder(); - int i = 0; - for (ColumnInfo ci : rr.getSignature()) { - b.put(ci.getInternalName(), i); - i++; - } - opPositionMap.put(rNode, b.build()); - } - - /* - * Why special handling for TableScan? - the RowResolver coming from hive - * for TScan still has all the columns, whereas the Optiq type we build is - * based on the needed columns in the TScanOp. - */ - void buildColumnMap(TableScanOperator tsOp, RelNode rNode) { - RelDataType oType = rNode.getRowType(); - int i = 0; - ImmutableMap.Builder b = new ImmutableMap.Builder(); - for (String fN : oType.getFieldNames()) { - b.put(fN, i); - i++; - } - opPositionMap.put(rNode, b.build()); - } - - Map reducerMap(Map inpMap, ReduceSinkOperator rsOp) { - ImmutableMap.Builder b = new ImmutableMap.Builder(); - Map colExprMap = rsOp.getColumnExprMap(); - for (Map.Entry e : colExprMap.entrySet()) { - String inpCol = ((ExprNodeColumnDesc) e.getValue()).getColumn(); - b.put(e.getKey(), inpMap.get(inpCol)); - } - return b.build(); - } - - /* - * The Optiq JoinRel datatype is formed by combining the columns from its - * input RelNodes. Whereas the Hive RowResolver of the JoinOp contains only - * the columns needed by childOps. - */ - void buildColumnMap(JoinOperator jOp, HiveJoinRel jRel) throws SemanticException { - RowResolver rr = sA.getRowResolver(jOp); - QBJoinTree hTree = parseCtx.getJoinContext().get(jOp); - Map leftMap = opPositionMap.get(jRel.getLeft()); - Map rightMap = opPositionMap.get(jRel.getRight()); - leftMap = reducerMap(leftMap, (ReduceSinkOperator) jOp.getParentOperators().get(0)); - rightMap = reducerMap(rightMap, (ReduceSinkOperator) jOp.getParentOperators().get(1)); - int leftColCount = jRel.getLeft().getRowType().getFieldCount(); - ImmutableMap.Builder b = new ImmutableMap.Builder(); - for (Map.Entry> tableEntry : rr.getRslvMap() - .entrySet()) { - String table = tableEntry.getKey(); - LinkedHashMap cols = tableEntry.getValue(); - Map posMap = leftMap; - int offset = 0; - if (hTree.getRightAliases() != null) { - for (String rAlias : hTree.getRightAliases()) { - if (table.equals(rAlias)) { - posMap = rightMap; - offset = leftColCount; - break; - } - } - } - for (Map.Entry colEntry : cols.entrySet()) { - ColumnInfo ci = colEntry.getValue(); - ExprNodeDesc e = jOp.getColumnExprMap().get(ci.getInternalName()); - String cName = ((ExprNodeColumnDesc) e).getColumn(); - int pos = posMap.get(cName); - - b.put(ci.getInternalName(), pos + offset); - } - } - opPositionMap.put(jRel, b.build()); - } - - void propagatePosMap(RelNode node, RelNode parent) { - opPositionMap.put(node, opPositionMap.get(parent)); - } - - RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, final boolean flatten) - throws SemanticException { - return convertToOptiqExpr(expr, optiqOP, 0, flatten); - } - - RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, int offset, - final boolean flatten) throws SemanticException { - ImmutableMap posMap = opPositionMap.get(optiqOP); - RexNodeConverter c = new RexNodeConverter(cluster, optiqOP.getRowType(), posMap, offset, - flatten); - return c.convert(expr); - } - - RelNode getParentNode(Operator hiveOp, int i) { - Operator p = hiveOp.getParentOperators().get(i); - return p == null ? null : hiveOpToRelNode.get(p); - } - - } - - static class JoinProcessor implements NodeProcessor { - @Override - @SuppressWarnings("unchecked") - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - HiveRel left = (HiveRel) ctx.getParentNode((Operator) nd, 0); - HiveRel right = (HiveRel) ctx.getParentNode((Operator) nd, 1); - JoinOperator joinOp = (JoinOperator) nd; - JoinCondDesc[] jConds = joinOp.getConf().getConds(); - assert jConds.length == 1; - HiveJoinRel joinRel = convertJoinOp(ctx, joinOp, jConds[0], left, right); - ctx.buildColumnMap(joinOp, joinRel); - ctx.hiveOpToRelNode.put(joinOp, joinRel); - return joinRel; - } - - /* - * @todo: cleanup, for now just copied from HiveToOptiqRelConvereter - */ - private HiveJoinRel convertJoinOp(Context ctx, JoinOperator op, JoinCondDesc jc, - HiveRel leftRel, HiveRel rightRel) throws SemanticException { - HiveJoinRel joinRel; - Operator leftParent = op.getParentOperators().get(jc.getLeft()); - Operator rightParent = op.getParentOperators().get(jc.getRight()); - - if (leftParent instanceof ReduceSinkOperator && rightParent instanceof ReduceSinkOperator) { - List leftCols = ((ReduceSinkDesc) (leftParent.getConf())).getKeyCols(); - List rightCols = ((ReduceSinkDesc) (rightParent.getConf())).getKeyCols(); - RexNode joinPredicate = null; - JoinRelType joinType = JoinRelType.INNER; - int rightColOffSet = leftRel.getRowType().getFieldCount(); - - // TODO: what about semi join - switch (jc.getType()) { - case JoinDesc.INNER_JOIN: - joinType = JoinRelType.INNER; - break; - case JoinDesc.LEFT_OUTER_JOIN: - joinType = JoinRelType.LEFT; - break; - case JoinDesc.RIGHT_OUTER_JOIN: - joinType = JoinRelType.RIGHT; - break; - case JoinDesc.FULL_OUTER_JOIN: - joinType = JoinRelType.FULL; - break; - } - - int i = 0; - for (ExprNodeDesc expr : leftCols) { - List eqExpr = new LinkedList(); - eqExpr.add(ctx.convertToOptiqExpr(expr, leftRel, 0, false)); - eqExpr.add(ctx.convertToOptiqExpr(rightCols.get(i), rightRel, rightColOffSet, false)); - - RexNode eqOp = ctx.cluster.getRexBuilder().makeCall(SqlStdOperatorTable.EQUALS, eqExpr); - i++; - - if (joinPredicate == null) { - joinPredicate = eqOp; - } else { - List conjElements = new LinkedList(); - conjElements.add(joinPredicate); - conjElements.add(eqOp); - joinPredicate = ctx.cluster.getRexBuilder().makeCall(SqlStdOperatorTable.AND, - conjElements); - } - } - - // Translate non-joinkey predicate - Set>> filterExprSet = op.getConf().getFilters().entrySet(); - if (!filterExprSet.isEmpty()) { - RexNode eqExpr; - int colOffSet; - RelNode childRel; - Operator parentHiveOp; - int inputId; - - for (Entry> entry : filterExprSet) { - inputId = entry.getKey().intValue(); - if (inputId == 0) { - colOffSet = 0; - childRel = leftRel; - parentHiveOp = leftParent; - } else if (inputId == 1) { - colOffSet = rightColOffSet; - childRel = rightRel; - parentHiveOp = rightParent; - } else { - throw new RuntimeException("Invalid Join Input"); - } - - for (ExprNodeDesc expr : entry.getValue()) { - eqExpr = ctx.convertToOptiqExpr(expr, childRel, colOffSet, false); - List conjElements = new LinkedList(); - conjElements.add(joinPredicate); - conjElements.add(eqExpr); - joinPredicate = ctx.cluster.getRexBuilder().makeCall(SqlStdOperatorTable.AND, - conjElements); - } - } - } - - joinRel = HiveJoinRel.getJoin(ctx.cluster, leftRel, rightRel, joinPredicate, joinType, - false); - } else { - throw new RuntimeException("Right & Left of Join Condition columns are not equal"); - } - - return joinRel; - } - - } - - private static int convertExpr(Context ctx, RelNode input, ExprNodeDesc expr, - List extraExprs) throws SemanticException { - final RexNode rex = ctx.convertToOptiqExpr(expr, input, false); - final int index; - if (rex instanceof RexInputRef) { - index = ((RexInputRef) rex).getIndex(); - } else { - index = input.getRowType().getFieldCount() + extraExprs.size(); - extraExprs.add(rex); - } - return index; - } - - private static AggregateCall convertAgg(Context ctx, AggregationDesc agg, RelNode input, - ColumnInfo cI, List extraExprs) throws SemanticException { - final Aggregation aggregation = AGG_MAP.get(agg.getGenericUDAFName()); - if (aggregation == null) { - throw new AssertionError("agg not found: " + agg.getGenericUDAFName()); - } - - List argList = new ArrayList(); - RelDataType type = TypeConverter.convert(cI.getType(), ctx.cluster.getTypeFactory()); - if (aggregation.equals(SqlStdOperatorTable.AVG)) { - type = type.getField("sum", false).getType(); - } - for (ExprNodeDesc expr : agg.getParameters()) { - int index = convertExpr(ctx, input, expr, extraExprs); - argList.add(index); - } - - /* - * set the type to the first arg, it there is one; because the RTi set on - * Aggregation call assumes this is the output type. - */ - if (argList.size() > 0) { - RexNode rex = ctx.convertToOptiqExpr(agg.getParameters().get(0), input, false); - type = rex.getType(); - } - return new AggregateCall(aggregation, agg.getDistinct(), argList, type, null); - } - - static class FilterProcessor implements NodeProcessor { - @Override - @SuppressWarnings("unchecked") - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - HiveRel input = (HiveRel) ctx.getParentNode((Operator) nd, 0); - FilterOperator filterOp = (FilterOperator) nd; - RexNode convertedFilterExpr = ctx.convertToOptiqExpr(filterOp.getConf().getPredicate(), - input, true); - - // Flatten the condition otherwise Optiq chokes on assertion - // (FilterRelBase) - if (convertedFilterExpr instanceof RexCall) { - RexCall call = (RexCall) convertedFilterExpr; - convertedFilterExpr = ctx.cluster.getRexBuilder().makeCall(call.getType(), - call.getOperator(), RexUtil.flatten(call.getOperands(), call.getOperator())); - } - - HiveRel filtRel = new HiveFilterRel(ctx.cluster, ctx.cluster.traitSetOf(HiveRel.CONVENTION), - input, convertedFilterExpr); - ctx.propagatePosMap(filtRel, input); - ctx.hiveOpToRelNode.put(filterOp, filtRel); - return filtRel; - } - } - - static class SelectProcessor implements NodeProcessor { - @Override - @SuppressWarnings("unchecked") - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - HiveRel inputRelNode = (HiveRel) ctx.getParentNode((Operator) nd, 0); - SelectOperator selectOp = (SelectOperator) nd; - - List colLst = selectOp.getConf().getColList(); - List optiqColLst = new LinkedList(); - - for (ExprNodeDesc colExpr : colLst) { - optiqColLst.add(ctx.convertToOptiqExpr(colExpr, inputRelNode, false)); - } - - /* - * Hive treats names that start with '_c' as internalNames; so change the - * names so we don't run into this issue when converting back to Hive AST. - */ - List oFieldNames = Lists.transform(selectOp.getConf().getOutputColumnNames(), - new Function() { - @Override - public String apply(String hName) { - return "_o_" + hName; - } - }); - - HiveRel selRel = HiveProjectRel.create(inputRelNode, optiqColLst, oFieldNames); - ctx.buildColumnMap(selectOp, selRel); - ctx.hiveOpToRelNode.put(selectOp, selRel); - return selRel; - } - } - - static class LimitProcessor implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - HiveRel input = (HiveRel) ctx.getParentNode((Operator) nd, 0); - LimitOperator limitOp = (LimitOperator) nd; - - // in Optiq, a limit is represented as a sort on 0 columns - final RexNode fetch; - if (limitOp.getConf().getLimit() >= 0) { - fetch = ctx.cluster.getRexBuilder().makeExactLiteral( - BigDecimal.valueOf(limitOp.getConf().getLimit())); - } else { - fetch = null; - } - RelTraitSet traitSet = ctx.cluster.traitSetOf(HiveRel.CONVENTION); - RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.EMPTY); - HiveRel sortRel = new HiveSortRel(ctx.cluster, traitSet, input, canonizedCollation, null, - fetch); - ctx.propagatePosMap(sortRel, input); - ctx.hiveOpToRelNode.put(limitOp, sortRel); - return sortRel; - } - } - - static class GroupByProcessor implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - - HiveRel input = (HiveRel) ctx.getParentNode((Operator) nd, 0); - GroupByOperator groupByOp = (GroupByOperator) nd; - RowResolver rr = ctx.sA.getRowResolver(groupByOp); - ArrayList signature = rr.getRowSchema().getSignature(); - - // GroupBy is represented by two operators, one map side and one reduce - // side. We only translate the map-side one. - if (groupByOp.getParentOperators().get(0) instanceof ReduceSinkOperator) { - ctx.hiveOpToRelNode.put(groupByOp, input); - return input; - } - - final List extraExprs = Lists.newArrayList(); - final BitSet groupSet = new BitSet(); - for (ExprNodeDesc key : groupByOp.getConf().getKeys()) { - int index = convertExpr(ctx, input, key, extraExprs); - groupSet.set(index); - } - List aggregateCalls = Lists.newArrayList(); - int i = groupByOp.getConf().getKeys().size(); - for (AggregationDesc agg : groupByOp.getConf().getAggregators()) { - aggregateCalls.add(convertAgg(ctx, agg, input, signature.get(i++), extraExprs)); - } - - if (!extraExprs.isEmpty()) { - // noinspection unchecked - input = HiveProjectRel.create(input, CompositeList.of(Lists.transform(input.getRowType() - .getFieldList(), new Function() { - @Override - public RexNode apply(RelDataTypeField input) { - return new RexInputRef(input.getIndex(), input.getType()); - } - }), extraExprs), null); - } - try { - HiveRel aggregateRel = new HiveAggregateRel(ctx.cluster, - ctx.cluster.traitSetOf(HiveRel.CONVENTION), input, groupSet, aggregateCalls); - ctx.buildColumnMap(groupByOp, aggregateRel); - ctx.hiveOpToRelNode.put(groupByOp, aggregateRel); - return aggregateRel; - } catch (InvalidRelException e) { - throw new AssertionError(e); // not possible - } - } - } - - static class ReduceSinkProcessor implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - HiveRel input = (HiveRel) ctx.getParentNode((Operator) nd, 0); - ReduceSinkOperator sinkOp = (ReduceSinkOperator) nd; - - // It is a sort reducer if and only if the number of reducers is 1. - final ReduceSinkDesc conf = sinkOp.getConf(); - if (conf.getNumReducers() != 1) { - Operator op = (Operator) nd; - ctx.hiveOpToRelNode.put(op, input); - return input; - } - - final String order = conf.getOrder(); // "+-" means "ASC, DESC" - assert order.length() == conf.getKeyCols().size(); - - /* - * numReducers == 1 and order.length = 1 => a RS for CrossJoin. - */ - if (order.length() == 0) { - Operator op = (Operator) nd; - ctx.hiveOpToRelNode.put(op, input); - return input; - } - - final List fieldCollations = Lists.newArrayList(); - final List extraExprs = Lists.newArrayList(); - for (Pair pair : Pair.zip(conf.getKeyCols(), - Lists.charactersOf(order))) { - int index = convertExpr(ctx, input, pair.left, extraExprs); - RelFieldCollation.Direction direction = getDirection(pair.right); - fieldCollations.add(new RelFieldCollation(index, direction)); - } - - if (!extraExprs.isEmpty()) { - // noinspection unchecked - input = HiveProjectRel.create(input, CompositeList.of(Lists.transform(input.getRowType() - .getFieldList(), new Function() { - @Override - public RexNode apply(RelDataTypeField input) { - return new RexInputRef(input.getIndex(), input.getType()); - } - }), extraExprs), null); - } - - RelTraitSet traitSet = ctx.cluster.traitSetOf(HiveRel.CONVENTION); - RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(fieldCollations)); - HiveRel sortRel = new HiveSortRel(ctx.cluster, traitSet, input, canonizedCollation, null, - null); - ctx.propagatePosMap(sortRel, input); - ctx.hiveOpToRelNode.put(sinkOp, sortRel); - - // REVIEW: Do we need to remove the columns we added due to extraExprs? - - return sortRel; - } - - private RelFieldCollation.Direction getDirection(char c) { - switch (c) { - case '+': - return RelFieldCollation.Direction.ASCENDING; - case '-': - return RelFieldCollation.Direction.DESCENDING; - default: - throw new AssertionError("unexpected direction " + c); - } - } - } - - static class TableScanProcessor implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - Context ctx = (Context) procCtx; - TableScanOperator tableScanOp = (TableScanOperator) nd; - RowResolver rr = ctx.sA.getRowResolver(tableScanOp); - - List neededCols = new ArrayList(tableScanOp.getNeededColumns()); - Statistics stats = tableScanOp.getStatistics(); - - try { - stats = addPartitionColumns(ctx, tableScanOp, tableScanOp.getConf().getAlias(), - ctx.sA.getTable(tableScanOp), stats, neededCols); - } catch (CloneNotSupportedException ce) { - throw new SemanticException(ce); - } - - if (stats.getColumnStats().size() != neededCols.size()) { - throw new SemanticException("Incomplete Col stats for table: " - + tableScanOp.getConf().getAlias()); - } - RelDataType rowType = TypeConverter.getType(ctx.cluster, rr, neededCols); - RelOptHiveTable optTable = new RelOptHiveTable(ctx.schema, tableScanOp.getConf().getAlias(), - rowType, ctx.sA.getTable(tableScanOp), null, null, null, null, null); - TableAccessRelBase tableRel = new HiveTableScanRel(ctx.cluster, - ctx.cluster.traitSetOf(HiveRel.CONVENTION), optTable, rowType); - ctx.buildColumnMap(tableScanOp, tableRel); - ctx.hiveOpToRelNode.put(tableScanOp, tableRel); - return tableRel; - } - - /* - * Add partition columns to needed columns and fake the COlStats for it. - */ - private Statistics addPartitionColumns(Context ctx, TableScanOperator tableScanOp, - String tblAlias, Table tbl, Statistics stats, List neededCols) - throws CloneNotSupportedException { - if (!tbl.isPartitioned()) { - return stats; - } - List pStats = new ArrayList(); - List pCols = tbl.getPartCols(); - for (FieldSchema pC : pCols) { - neededCols.add(pC.getName()); - ColStatistics cStats = stats.getColumnStatisticsForColumn(tblAlias, pC.getName()); - if (cStats == null) { - PrunedPartitionList partList = ctx.parseCtx.getOpToPartList().get(tableScanOp); - cStats = new ColStatistics(tblAlias, pC.getName(), pC.getType()); - cStats.setCountDistint(partList.getPartitions().size()); - pStats.add(cStats); - } - } - if (pStats.size() > 0) { - stats = stats.clone(); - stats.addToColumnStats(pStats); - } - - return stats; - } - } - - static class DefaultProcessor implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - @SuppressWarnings("unchecked") - Operator op = (Operator) nd; - Context ctx = (Context) procCtx; - RelNode node = ctx.getParentNode(op, 0); - ctx.hiveOpToRelNode.put(op, node); - return node; - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java index 45f8b8c..7a1e259 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; import java.math.BigDecimal; @@ -10,6 +27,8 @@ import java.util.List; import java.util.Map; +import net.hydromatic.avatica.ByteString; + import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -28,12 +47,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -204,7 +221,7 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { // || (udf instanceof GenericUDFToUnixTimeStamp) || (udf instanceof // GenericUDFTimestamp) || castExprUsingUDFBridge(udf)) { - castExpr = m_cluster.getRexBuilder().makeCast( + castExpr = m_cluster.getRexBuilder().makeAbstractCast( TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()), childRexNodeLst.get(0)); } @@ -264,7 +281,9 @@ protected RexNode convert(ExprNodeConstantDesc literal) { optiqLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue()); break; case BYTE: - optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value)); + byte[] byteArray = new byte[] { (Byte) value}; + ByteString bs = new ByteString(byteArray); + optiqLiteral = rexBuilder.makeBinaryLiteral(bs); break; case SHORT: optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java index c69f7ef..ab13ab3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; import java.lang.annotation.Annotation; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java index 755b5ce..3a25186 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.translator; import java.util.LinkedList; @@ -22,7 +39,6 @@ import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeFactory; -import org.eigenbase.reltype.RelDataTypeFactoryImpl.JavaType; import org.eigenbase.reltype.RelDataTypeField; import org.eigenbase.rex.RexBuilder; import org.eigenbase.sql.type.SqlTypeName; @@ -33,7 +49,7 @@ import com.google.common.collect.Lists; public class TypeConverter { - private static final Map m_optiqToHiveTypeNameMap; + private static final Map optiqToHiveTypeNameMap; // TODO: Handling of char[], varchar[], string... static { @@ -48,7 +64,7 @@ b.put(SqlTypeName.DATE.getName(), new HiveToken(HiveParser.TOK_DATE, "TOK_DATE")); b.put(SqlTypeName.TIMESTAMP.getName(), new HiveToken(HiveParser.TOK_TIMESTAMP, "TOK_TIMESTAMP")); b.put(SqlTypeName.BINARY.getName(), new HiveToken(HiveParser.TOK_BINARY, "TOK_BINARY")); - m_optiqToHiveTypeNameMap = b.build(); + optiqToHiveTypeNameMap = b.build(); }; /*********************** Convert Hive Types To Optiq Types ***********************/ @@ -297,7 +313,7 @@ public static HiveToken hiveToken(RelDataType optiqType) { } break; default: - ht = m_optiqToHiveTypeNameMap.get(optiqType.getSqlTypeName().getName()); + ht = optiqToHiveTypeNameMap.get(optiqType.getSqlTypeName().getName()); } return ht; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d91a83a..396a74a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -36,7 +36,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -114,7 +113,6 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException; -import org.apache.hadoop.hive.ql.optimizer.optiq.Pair; import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveVolcanoPlanner; @@ -276,6 +274,7 @@ import org.eigenbase.sql.SqlLiteral; import org.eigenbase.util.CompositeList; import org.eigenbase.util.ImmutableIntList; +import org.eigenbase.util.Pair; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -12137,7 +12136,7 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode ImmutableList.Builder bldr = new ImmutableList.Builder(); bldr.add(unionLeftInput); bldr.add(unionRightInput); - unionRel = new HiveUnionRel(m_cluster, TraitsUtil.getUnionTraitSet(m_cluster, null), + unionRel = new HiveUnionRel(m_cluster, TraitsUtil.getDefaultTraitSet(m_cluster), bldr.build()); m_relToHiveRR.put(unionRel, unionoutRR); @@ -13244,11 +13243,11 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticExce for (WindowExpressionSpec wExprSpec : windowExpressions) { if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) { Pair wtp = genWindowingProj(qb, wExprSpec, srcRel); - projsForWindowSelOp.add(wtp.getFirst()); + projsForWindowSelOp.add(wtp.getKey()); // 6.2.2 Update Output Row Schema ColumnInfo oColInfo = new ColumnInfo( - getColumnInternalName(projsForWindowSelOp.size()), wtp.getSecond(), null, false); + getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); String colAlias = wExprSpec.getAlias(); if (false) { out_rwsch.checkColumn(null, wExprSpec.getAlias()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java index adf55c8..129784d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; @@ -52,6 +53,11 @@ * otherwise it returns expr3. IF() returns a numeric or string value, depending * on the context in which it is used. */ +@Description( + name = "if", + value = "IF(expr1,expr2,expr3) - If expr1 is TRUE (expr1 <> 0 and expr1 <> NULL) then" + + " IF() returns expr2; otherwise it returns expr3. IF() returns a numeric or string value," + + " depending on the context in which it is used.") @VectorizedExpressions({ IfExprLongColumnLongColumn.class, IfExprDoubleColumnDoubleColumn.class, IfExprLongColumnLongScalar.class, IfExprDoubleColumnDoubleScalar.class,