diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 9b70448..3907340 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -641,8 +641,7 @@ "How many rows in the joining tables (except the streaming table) should be cached in memory."), // CBO related - HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations" - + " using Optiq framework."), + HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Optiq framework."), // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java index c47de8d..cdcec43 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java @@ -37,8 +37,7 @@ * more efficient. */ @Description(name = "example_group_concat", -value = "_FUNC_(col) - Example UDAF that concatenates all arguments from different rows into a" - + " single string") +value = "_FUNC_(col) - Example UDAF that concatenates all arguments from different rows into a single string") public class UDAFExampleGroupConcat extends UDAF { /** diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java index 2aeb361..ccb36d0 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java @@ -25,8 +25,7 @@ /** * Returns the max N double values. */ -@Description(name = "example_max_n", value = "_FUNC_(expr) -" - + " Example UDAF that returns the max N double values") +@Description(name = "example_max_n", value = "_FUNC_(expr) - Example UDAF that returns the max N double values") public class UDAFExampleMaxN extends UDAF { /** diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java index b250064..947167a 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java @@ -25,8 +25,7 @@ /** * Returns the min N double values. */ -@Description(name = "example_min_n", value = "_FUNC_(expr) - " - + "Example UDAF that returns the min N double values") +@Description(name = "example_min_n", value = "_FUNC_(expr) - Example UDAF that returns the min N double values") public class UDAFExampleMinN extends UDAF{ /** diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java index 8f67b5b..bc54e3c 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java @@ -24,8 +24,7 @@ * UDFExampleFormat. * */ -@Description(name = "example_format", value = "_FUNC_(expr) - " - + "Example UDAF that returns formatted String") +@Description(name = "example_format", value = "_FUNC_(expr) - Example UDAF that returns formated String") public class UDFExampleFormat extends UDF { public String evaluate(String format, Object... args) { diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java index afdee60..6b7360f 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java @@ -29,7 +29,7 @@ * */ @Description(name = "example_mapconcat", -value = "_FUNC_(expr) - Example UDAF that returns contents of Map as a formatted String") +value = "_FUNC_(expr) - Example UDAF that returns contents of Map as a formated String") public class UDFExampleMapConcat extends UDF { public String evaluate(Map a) { diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java index 9c42fac..8094946 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java @@ -36,7 +36,7 @@ * */ @Description(name = "udtfCount2", -value = "_FUNC_(col) - UDTF outputs the number of rows seen, twice.") +value = "_FUNC_(col) - UDF outputs the number of rows seen, twice.") public class GenericUDTFCount2 extends GenericUDTF { private transient Integer count = Integer.valueOf(0); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java index 5163f47..f9842fa 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java @@ -34,7 +34,7 @@ * */ @Description(name = "test_max", -value = "_FUNC_(col) - UDAF to report Max Value") +value = "_FUNC_(col) - UDF to report Max Value") public class UDAFTestMax extends UDAF { /** diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java index 857e485..382fa44 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java @@ -25,7 +25,7 @@ * A UDF for testing, which throws RuntimeException if the length of a string. */ @Description(name = "test_error", -value = "_FUNC_(col) - UDF throws RuntimeException if expression evaluates to false") +value = "_FUNC_(col) - UDF throws RuntimeException if expression evaluates to false") public class UDFTestErrorOnFalse extends UDF { public int evaluate(Boolean b) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java index e0707df..5fa63f1 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java @@ -27,7 +27,6 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Reporter; - @Description(name = "counter", value = "_FUNC_(col) - UDF to report MR counter values") public class DummyContextUDF extends GenericUDF { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 6a32c25..5dab171 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -51,19 +51,18 @@ private int noOfJoins = 0; private int noOfOuterJoins = 0; private boolean hasLateralViews; - + private boolean multiDestQuery; private boolean filterWithSubQuery; - + public boolean hasJoin() { - return noOfJoins > 0; + return (noOfJoins > 0); } public void incrementJoinCount(boolean outerJoin) { noOfJoins++; - if (outerJoin) { + if (outerJoin) noOfOuterJoins++; - } } public int getJoinCount() { @@ -171,7 +170,7 @@ public void setHasMapGroupBy(boolean hasMapGroupBy) { } public boolean hasMultiDestQuery() { - return multiDestQuery; + return this.multiDestQuery; } public void setMultiDestQuery(boolean multiDestQuery) { @@ -205,7 +204,7 @@ public void clear() { noOfJoins = 0; noOfOuterJoins = 0; - + multiDestQuery = false; filterWithSubQuery = false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java index 5369500..e9e052f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java @@ -27,7 +27,7 @@ import org.eigenbase.rel.metadata.DefaultRelMetadataProvider; import org.eigenbase.rel.metadata.RelMetadataProvider; -public final class HiveDefaultRelMetadataProvider { +public class HiveDefaultRelMetadataProvider { private HiveDefaultRelMetadataProvider() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index 3ecce65..2ad5c34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -23,16 +23,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; +import java.util.Map.Entry; -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; import org.eigenbase.rel.RelFactories.ProjectFactory; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptUtil; @@ -46,6 +42,12 @@ import org.eigenbase.sql.validate.SqlValidatorUtil; import org.eigenbase.util.Pair; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; + /** * Generic utility functions needed for Optiq based Hive CBO. */ @@ -72,7 +74,7 @@ return vCols; } - public static boolean validateASTForCBO(ASTNode ast) { + public static boolean validateASTForCBO (ASTNode ast) { String astTree = ast.toStringTree(); String [] tokens = {"TOK_CHARSETLITERAL"}; for (String token : tokens) { @@ -84,13 +86,14 @@ public static boolean validateASTForCBO(ASTNode ast) { } public static List getProjsFromBelowAsInputRef(final RelNode rel) { - return Lists.transform(rel.getRowType().getFieldList(), + List projectList = Lists.transform(rel.getRowType().getFieldList(), new Function() { @Override public RexNode apply(RelDataTypeField field) { return rel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); } }); + return projectList; } public static List translateBitSetToProjIndx(BitSet projBitSet) { @@ -134,6 +137,7 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel RelNode leftRel = inputRels[0]; RelNode rightRel = inputRels[1]; RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder(); + RexNode outJoinCond = null; int origLeftInputSize = leftRel.getRowType().getFieldCount(); int origRightInputSize = rightRel.getRowType().getFieldCount(); @@ -176,7 +180,6 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel } } - RexNode outJoinCond = null; for (i = 0; i < origColEqConds.size(); i++) { Pair p = origColEqConds.get(i); RexNode leftKey = leftJoinKeys.get(i); @@ -245,14 +248,12 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel * */ public static class JoinPredicateInfo { - - private final ImmutableList nonEquiJoinPredicateElements; - private final ImmutableList equiJoinPredicateElements; - private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; - private final ImmutableMap> - mapOfProjIndxInJoinSchemaToLeafPInfo; + private final ImmutableList nonEquiJoinPredicateElements; + private final ImmutableList equiJoinPredicateElements; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; + private final ImmutableMap> mapOfProjIndxInJoinSchemaToLeafPInfo; public JoinPredicateInfo(List nonEquiJoinPredicateElements, List equiJoinPredicateElements, @@ -310,28 +311,31 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { } public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNode predicate) { + JoinPredicateInfo jpi = null; + JoinLeafPredicateInfo jlpi = null; List equiLPIList = new ArrayList(); List nonEquiLPIList = new ArrayList(); Set projsFromLeftPartOfJoinKeys = new HashSet(); Set projsFromRightPartOfJoinKeys = new HashSet(); Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); - Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = - new HashMap>(); - Map> mapOfProjIndxInJoinSchemaToLeafPInfo = - new HashMap>(); + Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); + Map> mapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); + List tmpJLPILst = null; int rightOffSet = j.getLeft().getRowType().getFieldCount(); + int projIndxInJoin; + List conjuctiveElements; // 1. Decompose Join condition to a number of leaf predicates // (conjuctive elements) - List conjuctiveElements = RelOptUtil.conjunctions(predicate); + conjuctiveElements = RelOptUtil.conjunctions(predicate); // 2. Walk through leaf predicates building up JoinLeafPredicateInfo for (RexNode ce : conjuctiveElements) { // 2.1 Construct JoinLeafPredicateInfo - JoinLeafPredicateInfo jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); + jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); // 2.2 Classify leaf predicate as Equi vs Non Equi - if (jlpi.comparisonType == SqlKind.EQUALS) { + if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { equiLPIList.add(jlpi); } else { nonEquiLPIList.add(jlpi); @@ -346,12 +350,10 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod // 2.4 Update Join Key to JoinLeafPredicateInfo map with keys // from left - List tmpJLPILst; for (Integer projIndx : jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) { tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndx); - if (tmpJLPILst == null) { + if (tmpJLPILst == null) tmpJLPILst = new ArrayList(); - } tmpJLPILst.add(jlpi); tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst); } @@ -359,11 +361,10 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod // 2.5 Update Join Key to JoinLeafPredicateInfo map with keys // from right for (Integer projIndx : jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()) { - int projIndxInJoin = projIndx + rightOffSet; + projIndxInJoin = projIndx + rightOffSet; tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndxInJoin); - if (tmpJLPILst == null) { + if (tmpJLPILst == null) tmpJLPILst = new ArrayList(); - } tmpJLPILst.add(jlpi); tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndxInJoin, tmpJLPILst); } @@ -378,9 +379,10 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod } // 4. Construct JoinPredicateInfo - return new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, - projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, - mapOfProjIndxInJoinSchemaToLeafPInfo); + jpi = new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, + projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, + mapOfProjIndxInJoinSchemaToLeafPInfo); + return jpi; } } @@ -395,39 +397,38 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod * of equi join keys; the indexes are both in child and Join node schema.
*/ public static class JoinLeafPredicateInfo { - - private final SqlKind comparisonType; - private final ImmutableList joinKeyExprsFromLeft; - private final ImmutableList joinKeyExprsFromRight; - private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; + private final SqlKind m_comparisonType; + private final ImmutableList m_joinKeyExprsFromLeft; + private final ImmutableList m_joinKeyExprsFromRight; + private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInJoinSchema) { - this.comparisonType = comparisonType; - this.joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); - this.joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); - this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + m_comparisonType = comparisonType; + m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); + m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); + m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); } public List getJoinKeyExprsFromLeft() { - return joinKeyExprsFromLeft; + return m_joinKeyExprsFromLeft; } public List getJoinKeyExprsFromRight() { - return joinKeyExprsFromRight; + return m_joinKeyExprsFromRight; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return m_projsFromLeftPartOfJoinKeysInChildSchema; } /** @@ -436,18 +437,19 @@ public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsF * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return m_projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return projsFromRightPartOfJoinKeysInChildSchema; + return m_projsFromRightPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return projsFromRightPartOfJoinKeysInJoinSchema; + return m_projsFromRightPartOfJoinKeysInJoinSchema; } private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { + JoinLeafPredicateInfo jlpi = null; List filterNulls = new ArrayList(); List joinKeyExprsFromLeft = new ArrayList(); List joinKeyExprsFromRight = new ArrayList(); @@ -457,9 +459,9 @@ private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel int rightOffSet = j.getLeft().getRowType().getFieldCount(); // 1. Split leaf join predicate to expressions from left, right - - RelOptUtil.splitJoinCondition(j.getSystemFieldList(), j.getLeft(), j.getRight(), pe, - joinKeyExprsFromLeft, joinKeyExprsFromRight, filterNulls, null); + @SuppressWarnings("unused") + RexNode nonEquiPredicate = RelOptUtil.splitJoinCondition(j.getSystemFieldList(), j.getLeft(), + j.getRight(), pe, joinKeyExprsFromLeft, joinKeyExprsFromRight, filterNulls, null); // 2. For left expressions, collect child projection indexes used InputReferencedVisitor irvLeft = new InputReferencedVisitor(); @@ -478,10 +480,11 @@ private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel } // 4. Construct JoinLeafPredicateInfo + jlpi = new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, + projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, + projsFromRightPartOfJoinKeysInJoinSchema); - return new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, - projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, - projsFromRightPartOfJoinKeysInJoinSchema); + return jlpi; } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveTypeSystemImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveTypeSystemImpl.java index 69c7b8b..1bc5a2c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveTypeSystemImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveTypeSystemImpl.java @@ -35,7 +35,7 @@ public int getMaxScale(SqlTypeName typeName) { switch (typeName) { case DECIMAL: - return MAX_DECIMAL_SCALE; + return getMaxNumericScale(); case INTERVAL_DAY_TIME: case INTERVAL_YEAR_MONTH: return SqlTypeName.MAX_INTERVAL_FRACTIONAL_SECOND_PRECISION; @@ -70,7 +70,7 @@ public int getDefaultPrecision(SqlTypeName typeName) { public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: - return MAX_DECIMAL_PRECISION; + return getMaxNumericPrecision(); case VARCHAR: return MAX_VARCHAR_PRECISION; case CHAR: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java index a2724cc..d2b08fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java @@ -30,6 +30,7 @@ private static final long serialVersionUID = 1L; public OptiqSemanticException() { + super(); } public OptiqSemanticException(String message) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java index 73f1c0b..50acf42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java @@ -54,26 +54,26 @@ import com.google.common.collect.ImmutableMap.Builder; public class RelOptHiveTable extends RelOptAbstractTable { - - private final Table hiveTblMetadata; - private final ImmutableList hiveNonPartitionCols; + private final Table hiveTblMetadata; + private final ImmutableList hiveNonPartitionCols; private final ImmutableMap hiveNonPartitionColsMap; private final ImmutableMap hivePartitionColsMap; - private final int noOfProjs; - private final HiveConf hiveConf; + private final int noOfProjs; + final HiveConf hiveConf; - private double rowCount = -1; - private final Map hiveColStatsMap = new HashMap(); - private PrunedPartitionList partitionList; - private final Map partitionCache; - private final AtomicInteger noColsMissingStats; + private double rowCount = -1; + Map hiveColStatsMap = new HashMap(); + PrunedPartitionList partitionList; + Map partitionCache; + AtomicInteger noColsMissingStats; - private static final Log LOG = LogFactory.getLog(RelOptHiveTable.class.getName()); + protected static final Log LOG = LogFactory + .getLog(RelOptHiveTable.class + .getName()); public RelOptHiveTable(RelOptSchema optiqSchema, String name, RelDataType rowType, - Table hiveTblMetadata, List hiveNonPartitionCols, - List hivePartitionCols, HiveConf hconf, - Map partitionCache, AtomicInteger noColsMissingStats) { + Table hiveTblMetadata, List hiveNonPartitionCols, + List hivePartitionCols, HiveConf hconf, Map partitionCache, AtomicInteger noColsMissingStats) { super(optiqSchema, name, rowType); this.hiveTblMetadata = hiveTblMetadata; this.hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols); @@ -87,7 +87,7 @@ public RelOptHiveTable(RelOptSchema optiqSchema, String name, RelDataType rowTyp private static ImmutableMap getColInfoMap(List hiveCols, int startIndx) { - Builder bldr = ImmutableMap.builder(); + Builder bldr = ImmutableMap. builder(); int indx = startIndx; for (ColumnInfo ci : hiveCols) { @@ -131,9 +131,8 @@ public double getRowCount() { } } - if (rowCount == -1) { + if (rowCount == -1) noColsMissingStats.getAndIncrement(); - } return rowCount; } @@ -143,14 +142,14 @@ public Table getHiveTableMD() { } private String getColNamesForLogging(Set colLst) { - StringBuilder sb = new StringBuilder(); + StringBuffer sb = new StringBuffer(); boolean firstEntry = true; for (String colName : colLst) { if (firstEntry) { sb.append(colName); firstEntry = false; } else { - sb.append(", ").append(colName); + sb.append(", " + colName); } } return sb.toString(); @@ -159,20 +158,16 @@ private String getColNamesForLogging(Set colLst) { public void computePartitionList(HiveConf conf, RexNode pruneNode) { try { - if (!hiveTblMetadata.isPartitioned() || pruneNode == null || - InputFinder.bits(pruneNode).isEmpty()) { + if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0 ) { // there is no predicate on partitioning column, we need all partitions in this case. - partitionList = - PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); + partitionList = PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); return; } // We have valid pruning expressions, only retrieve qualifying partitions - ExprNodeDesc pruneExpr = - pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true)); + ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true)); - partitionList = - PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); + partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); } catch (HiveException he) { throw new RuntimeException(he); } @@ -186,9 +181,9 @@ private void updateColStats(Set projIndxLst) { Set colNamesFailedStats = new HashSet(); // 1. Separate required columns to Non Partition and Partition Cols + ColumnInfo tmp; for (Integer pi : projIndxLst) { if (hiveColStatsMap.get(pi) == null) { - ColumnInfo tmp; if ((tmp = hiveNonPartitionColsMap.get(pi)) != null) { nonPartColNamesThatRqrStats.add(tmp.getInternalName()); nonPartColIndxsThatRqrStats.add(pi); @@ -212,10 +207,29 @@ private void updateColStats(Set projIndxLst) { } // 2. Obtain Col Stats for Non Partition Cols - if (!nonPartColNamesThatRqrStats.isEmpty()) { + if (nonPartColNamesThatRqrStats.size() > 0) { List hiveColStats; - if (hiveTblMetadata.isPartitioned()) { + if (!hiveTblMetadata.isPartitioned()) { + // 2.1 Handle the case for unpartitioned table. + hiveColStats = StatsUtils.getTableColumnStats(hiveTblMetadata, hiveNonPartitionCols, + nonPartColNamesThatRqrStats); + + // 2.1.1 Record Column Names that we needed stats for but couldn't + if (hiveColStats == null) { + colNamesFailedStats.addAll(nonPartColNamesThatRqrStats); + } else if (hiveColStats.size() != nonPartColNamesThatRqrStats.size()) { + Set setOfFiledCols = new HashSet(nonPartColNamesThatRqrStats); + + Set setOfObtainedColStats = new HashSet(); + for (ColStatistics cs : hiveColStats) { + setOfObtainedColStats.add(cs.getColumnName()); + } + setOfFiledCols.removeAll(setOfObtainedColStats); + + colNamesFailedStats.addAll(setOfFiledCols); + } + } else { // 2.2 Obtain col stats for partitioned table. try { if (partitionList.getNotDeniedPartns().isEmpty()) { @@ -228,44 +242,24 @@ private void updateColStats(Set projIndxLst) { } colNamesFailedStats.clear(); } else { - Statistics stats = - StatsUtils.collectStatistics(hiveConf, partitionList, hiveTblMetadata, - hiveNonPartitionCols, nonPartColNamesThatRqrStats, true, true); - rowCount = stats.getNumRows(); - hiveColStats = new ArrayList(); - for (String c : nonPartColNamesThatRqrStats) { - ColStatistics cs = stats.getColumnStatisticsFromColName(c); - if (cs != null) { - hiveColStats.add(cs); - } else { - colNamesFailedStats.add(c); - } + Statistics stats = StatsUtils.collectStatistics(hiveConf, partitionList, + hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats, true, true); + rowCount = stats.getNumRows(); + hiveColStats = new ArrayList(); + for (String c : nonPartColNamesThatRqrStats) { + ColStatistics cs = stats.getColumnStatisticsFromColName(c); + if (cs != null) { + hiveColStats.add(cs); + } else { + colNamesFailedStats.add(c); } } + } } catch (HiveException e) { String logMsg = "Collecting stats failed."; LOG.error(logMsg); throw new RuntimeException(logMsg); } - } else { - // 2.1 Handle the case for unpartitioned table. - hiveColStats = StatsUtils.getTableColumnStats(hiveTblMetadata, hiveNonPartitionCols, - nonPartColNamesThatRqrStats); - - // 2.1.1 Record Column Names that we needed stats for but couldn't - if (hiveColStats == null) { - colNamesFailedStats.addAll(nonPartColNamesThatRqrStats); - } else if (hiveColStats.size() != nonPartColNamesThatRqrStats.size()) { - Set setOfFiledCols = new HashSet(nonPartColNamesThatRqrStats); - - Set setOfObtainedColStats = new HashSet(); - for (ColStatistics cs : hiveColStats) { - setOfObtainedColStats.add(cs.getColumnName()); - } - setOfFiledCols.removeAll(setOfObtainedColStats); - - colNamesFailedStats.addAll(setOfFiledCols); - } } if (hiveColStats != null && hiveColStats.size() == nonPartColNamesThatRqrStats.size()) { @@ -279,10 +273,10 @@ private void updateColStats(Set projIndxLst) { if (colNamesFailedStats.isEmpty() && !partColNamesThatRqrStats.isEmpty()) { ColStatistics cStats = null; for (int i = 0; i < partColNamesThatRqrStats.size(); i++) { - cStats = new ColStatistics(hiveTblMetadata.getTableName(), partColNamesThatRqrStats.get(i), - hivePartitionColsMap.get(partColIndxsThatRqrStats.get(i)).getTypeName()); - cStats.setCountDistint( - getDistinctCount(partitionList.getPartitions(), partColNamesThatRqrStats.get(i))); + cStats = new ColStatistics(hiveTblMetadata.getTableName(), + partColNamesThatRqrStats.get(i), hivePartitionColsMap.get( + partColIndxsThatRqrStats.get(i)).getTypeName()); + cStats.setCountDistint(getDistinctCount(partitionList.getPartitions(),partColNamesThatRqrStats.get(i))); hiveColStatsMap.put(partColIndxsThatRqrStats.get(i), cStats); } } @@ -306,7 +300,7 @@ private int getDistinctCount(Set partitions, String partColName) { } public List getColStat(List projIndxLst) { - ImmutableList.Builder colStatsBldr = ImmutableList.builder(); + ImmutableList.Builder colStatsBldr = ImmutableList. builder(); if (projIndxLst != null) { updateColStats(new HashSet(projIndxLst)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java index 2bd25ce..9b653d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java @@ -1,20 +1,3 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.hadoop.hive.ql.optimizer.optiq; @@ -24,12 +7,7 @@ import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; -public final class TraitsUtil { - - private TraitsUtil() { - throw new UnsupportedOperationException("Can't initialize class"); - } - +public class TraitsUtil { public static RelTraitSet getSortTraitSet(RelOptCluster cluster, RelTraitSet traitSet, RelCollation collation) { return traitSet.plus(collation); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java index e3ee4c1..a4df413 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java @@ -25,43 +25,44 @@ public class HiveCost implements RelOptCost { // ~ Static fields/initializers --------------------------------------------- - public static final HiveCost INFINITY = - new HiveCost(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY) { - @Override - public String toString() { - return "{inf}"; - } - }; - - public static final HiveCost HUGE = - new HiveCost(Double.MAX_VALUE, Double.MAX_VALUE, Double.MAX_VALUE) { - @Override - public String toString() { - return "{huge}"; - } - }; - - public static final HiveCost ZERO = new HiveCost(0.0, 0.0, 0.0) { - @Override - public String toString() { - return "{0}"; - } - }; - - public static final HiveCost TINY = new HiveCost(1.0, 1.0, 0.0) { - @Override - public String toString() { - return "{tiny}"; - } - }; - - public static final RelOptCostFactory FACTORY = new Factory(); + public static final HiveCost INFINITY = new HiveCost(Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY) { + @Override + public String toString() { + return "{inf}"; + } + }; + + public static final HiveCost HUGE = new HiveCost(Double.MAX_VALUE, Double.MAX_VALUE, + Double.MAX_VALUE) { + @Override + public String toString() { + return "{huge}"; + } + }; + + public static final HiveCost ZERO = new HiveCost(0.0, 0.0, 0.0) { + @Override + public String toString() { + return "{0}"; + } + }; + + public static final HiveCost TINY = new HiveCost(1.0, 1.0, 0.0) { + @Override + public String toString() { + return "{tiny}"; + } + }; + + public static final RelOptCostFactory FACTORY = new Factory(); // ~ Instance fields -------------------------------------------------------- - private final double cpu; - private final double io; - private final double rowCount; + final double cpu; + final double io; + final double rowCount; // ~ Constructors ----------------------------------------------------------- @@ -76,27 +77,21 @@ public String toString() { // ~ Methods ---------------------------------------------------------------- - @Override public double getCpu() { return cpu; } - @Override public boolean isInfinite() { - return (this == INFINITY) || - (this.rowCount == Double.POSITIVE_INFINITY) || - (this.cpu == Double.POSITIVE_INFINITY) || - (this.io == Double.POSITIVE_INFINITY); + return (this == INFINITY) || (this.rowCount == Double.POSITIVE_INFINITY) + || (this.cpu == Double.POSITIVE_INFINITY) || (this.io == Double.POSITIVE_INFINITY); } - @Override public double getIo() { return io; } // TODO: If two cost is equal, could we do any better than comparing // cardinality (may be some other heuristics to break the tie) - @Override public boolean isLe(RelOptCost other) { return this == other || this.rowCount <= other.getRows(); /* @@ -106,7 +101,6 @@ public boolean isLe(RelOptCost other) { */ } - @Override public boolean isLt(RelOptCost other) { return this.rowCount < other.getRows(); /* @@ -114,12 +108,10 @@ public boolean isLt(RelOptCost other) { */ } - @Override public double getRows() { return rowCount; } - @Override public boolean equals(RelOptCost other) { return (this == other) || ((this.rowCount) == (other.getRows())); @@ -129,7 +121,6 @@ public boolean equals(RelOptCost other) { */ } - @Override public boolean isEqWithEpsilon(RelOptCost other) { return (this == other) || (Math.abs((this.rowCount) - (other.getRows())) < RelOptUtil.EPSILON); /* @@ -138,17 +129,15 @@ public boolean isEqWithEpsilon(RelOptCost other) { */ } - @Override public RelOptCost minus(RelOptCost other) { if (this == INFINITY) { return this; } - return new HiveCost(this.rowCount - other.getRows(), this.cpu - other.getCpu(), - this.io - other.getIo()); + return new HiveCost(this.rowCount - other.getRows(), this.cpu - other.getCpu(), this.io + - other.getIo()); } - @Override public RelOptCost multiplyBy(double factor) { if (this == INFINITY) { return this; @@ -156,7 +145,6 @@ public RelOptCost multiplyBy(double factor) { return new HiveCost(rowCount * factor, cpu * factor, io * factor); } - @Override public double divideBy(RelOptCost cost) { // Compute the geometric average of the ratios of all of the factors // which are non-zero and finite. @@ -183,13 +171,12 @@ public double divideBy(RelOptCost cost) { return Math.pow(d, 1 / n); } - @Override public RelOptCost plus(RelOptCost other) { - if ((this == INFINITY) || other.isInfinite()) { + if ((this == INFINITY) || (other.isInfinite())) { return INFINITY; } - return new HiveCost(this.rowCount + other.getRows(), this.cpu + other.getCpu(), - this.io + other.getIo()); + return new HiveCost(this.rowCount + other.getRows(), this.cpu + other.getCpu(), this.io + + other.getIo()); } @Override @@ -197,29 +184,26 @@ public String toString() { return "{" + rowCount + " rows, " + cpu + " cpu, " + io + " io}"; } - private static final class Factory implements RelOptCostFactory { + private static class Factory implements RelOptCostFactory { + private Factory() { + } - @Override public RelOptCost makeCost(double rowCount, double cpu, double io) { return new HiveCost(rowCount, cpu, io); } - @Override public RelOptCost makeHugeCost() { return HUGE; } - @Override public HiveCost makeInfiniteCost() { return INFINITY; } - @Override public HiveCost makeTinyCost() { return TINY; } - @Override public HiveCost makeZeroCost() { return ZERO; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java index 878f493..257c380 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java @@ -22,14 +22,13 @@ import org.eigenbase.relopt.RelOptCost; public class HiveCostUtil { - - private static final double CPU_COST_IN_NANO_SEC = 1.0; - private static final double NET_COST_IN_NANO_SEC = 150 * CPU_COST_IN_NANO_SEC; - private static final double LOCAL_FS_WRITE_COST_IN_NANO_SEC = 4 * NET_COST_IN_NANO_SEC; - private static final double LOCAL_FS_READ_COST_IN_NANO_SEC = 4 * NET_COST_IN_NANO_SEC; - private static final double HDFS_WRITE_COST_IN_NANO_SEC = 10 * LOCAL_FS_WRITE_COST_IN_NANO_SEC; + private static final double cpuCostInNanoSec = 1.0; + private static final double netCostInNanoSec = 150 * cpuCostInNanoSec; + private static final double localFSWriteCostInNanoSec = 4 * netCostInNanoSec; + private static final double localFSReadCostInNanoSec = 4 * netCostInNanoSec; + private static final double hDFSWriteCostInNanoSec = 10 * localFSWriteCostInNanoSec; @SuppressWarnings("unused") - private static final double HDFS_READ_COST_IN_NANO_SEC = 1.5 * LOCAL_FS_READ_COST_IN_NANO_SEC; + private static final double hDFSReadCostInNanoSec = 1.5 * localFSReadCostInNanoSec; public static RelOptCost computCardinalityBasedCost(HiveRel hr) { return new HiveCost(hr.getRows(), 0, 0); @@ -37,6 +36,6 @@ public static RelOptCost computCardinalityBasedCost(HiveRel hr) { public static HiveCost computeCost(HiveTableScanRel t) { double cardinality = t.getRows(); - return new HiveCost(cardinality, 0, HDFS_WRITE_COST_IN_NANO_SEC * cardinality * 0); + return new HiveCost(cardinality, 0, hDFSWriteCostInNanoSec * cardinality * 0); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java index c0ed341..15596bc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java @@ -1,20 +1,3 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.hadoop.hive.ql.optimizer.optiq.cost; import org.eigenbase.rel.RelCollationTraitDef; @@ -24,7 +7,7 @@ /** * Refinement of {@link org.eigenbase.relopt.volcano.VolcanoPlanner} for Hive. - * + * *

* It uses {@link org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveCost} as * its cost model. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java index 0945c67..fc19895 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java @@ -75,8 +75,7 @@ public double getRows() { public RelNode createAggregate(RelNode child, BitSet groupSet, List aggCalls) { try { - return new HiveAggregateRel(child.getCluster(), child.getTraitSet(), child, groupSet, - aggCalls); + return new HiveAggregateRel(child.getCluster(), child.getTraitSet(), child, groupSet, aggCalls); } catch (InvalidRelException e) { throw new RuntimeException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java index 9ee24ce..8b85046 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java @@ -32,8 +32,7 @@ public static final FilterFactory DEFAULT_FILTER_FACTORY = new HiveFilterFactoryImpl(); - public HiveFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, - RexNode condition) { + public HiveFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); } @@ -54,13 +53,15 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { /** * Implementation of {@link FilterFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel}. + * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel} + * . */ private static class HiveFilterFactoryImpl implements FilterFactory { @Override public RelNode createFilter(RelNode child, RexNode condition) { RelOptCluster cluster = child.getCluster(); - return new HiveFilterRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); + HiveFilterRel filter = new HiveFilterRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); + return filter; } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java index 2749815..363e9ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java @@ -60,14 +60,19 @@ public static HiveJoinRel getJoin(RelOptCluster cluster, RelNode left, RelNode right, RexNode condition, JoinRelType joinType, boolean leftSemiJoin) { - Set variablesStopped = Collections.emptySet(); - return new HiveJoinRel(cluster, null, left, right, condition, joinType, variablesStopped, - JoinAlgorithm.NONE, null, leftSemiJoin); + try { + Set variablesStopped = Collections.emptySet(); + return new HiveJoinRel(cluster, null, left, right, condition, joinType, variablesStopped, + JoinAlgorithm.NONE, null, leftSemiJoin); + } catch (InvalidRelException e) { + throw new RuntimeException(e); + } } protected HiveJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, - RexNode condition, JoinRelType joinType, Set variablesStopped, JoinAlgorithm joinAlgo, - MapJoinStreamingRelation streamingSideForMapJoin, boolean leftSemiJoin) { + RexNode condition, JoinRelType joinType, Set variablesStopped, + JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin, boolean leftSemiJoin) + throws InvalidRelException { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), left, right, condition, joinType, variablesStopped); this.joinAlgorithm = joinAlgo; @@ -81,9 +86,15 @@ public void implement(Implementor implementor) { @Override public final HiveJoinRel copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { - Set variablesStopped = Collections.emptySet(); - return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType, - variablesStopped, JoinAlgorithm.NONE, null, leftSemiJoin); + try { + Set variablesStopped = Collections.emptySet(); + return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType, + variablesStopped, JoinAlgorithm.NONE, null, leftSemiJoin); + } catch (InvalidRelException e) { + // Semantic error not possible. Must be a bug. Convert to + // internal error. + throw new AssertionError(e); + } } public JoinAlgorithm getJoinAlgorithm() { @@ -112,7 +123,7 @@ public RelDataType deriveRowType() { if (leftSemiJoin) { return deriveJoinRowType(left.getRowType(), null, JoinRelType.INNER, getCluster().getTypeFactory(), null, - Collections.emptyList()); + Collections. emptyList()); } return super.deriveRowType(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java index 0926b7e..f8755d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java @@ -47,7 +47,6 @@ public HiveLimitRel copy(RelTraitSet traitSet, List newInputs) { return new HiveLimitRel(getCluster(), traitSet, sole(newInputs), offset, fetch); } - @Override public void implement(Implementor implementor) { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java index aea4382..c643aa4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java @@ -22,6 +22,7 @@ import java.util.List; import com.google.common.collect.ImmutableList; + import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException; import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; @@ -33,6 +34,7 @@ import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelOptCost; import org.eigenbase.relopt.RelOptPlanner; +import org.eigenbase.relopt.RelOptRule; import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeField; @@ -80,7 +82,7 @@ public HiveProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child * aliases of the expressions */ public static HiveProjectRel create(RelNode child, List exps, - List fieldNames) throws OptiqSemanticException { + List fieldNames) throws OptiqSemanticException{ RelOptCluster cluster = child.getCluster(); // 1 Ensure columnNames are unique - OPTIQ-411 @@ -89,14 +91,14 @@ public static HiveProjectRel create(RelNode child, List exps, throw new OptiqSemanticException(msg); } RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), exps, fieldNames); - return create(cluster, child, exps, rowType, Collections.emptyList()); + return create(cluster, child, exps, rowType, Collections. emptyList()); } /** * Creates a HiveProjectRel. */ - public static HiveProjectRel create(RelOptCluster cluster, RelNode child, - List exps, RelDataType rowType, final List collationList) { + public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, + RelDataType rowType, final List collationList) { RelTraitSet traitSet = TraitsUtil.getDefaultTraitSet(cluster); return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -104,9 +106,8 @@ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, /** * Creates a HiveProjectRel. */ - public static HiveProjectRel create(RelOptCluster cluster, RelNode child, - List exps, RelDataType rowType, RelTraitSet traitSet, - final List collationList) { + public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, + RelDataType rowType, RelTraitSet traitSet, final List collationList) { return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -137,8 +138,7 @@ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, * @return relational expression which projects a subset of the input fields * @throws OptiqSemanticException */ - public static RelNode projectMapping(RelNode rel, Mapping mapping, List fieldNames) - throws OptiqSemanticException { + public static RelNode projectMapping(RelNode rel, Mapping mapping, List fieldNames) throws OptiqSemanticException { assert mapping.getMappingType().isSingleSource(); assert mapping.getMappingType().isMandatorySource(); @@ -154,10 +154,9 @@ public static RelNode projectMapping(RelNode rel, Mapping mapping, List for (int i = 0; i < mapping.getTargetCount(); i++) { int source = mapping.getSource(i); final RelDataTypeField sourceField = fields.get(source); - outputNameList.add( - ((fieldNames == null) || (fieldNames.size() <= i) || (fieldNames.get(i) == null)) - ? sourceField.getName() - : fieldNames.get(i)); + outputNameList + .add(((fieldNames == null) || (fieldNames.size() <= i) || (fieldNames.get(i) == null)) ? sourceField + .getName() : fieldNames.get(i)); outputProjList.add(rexBuilder.makeInputRef(rel, source)); } @@ -186,19 +185,21 @@ public void implement(Implementor implementor) { /** * Implementation of {@link ProjectFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel}. + * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel} + * . */ private static class HiveProjectFactoryImpl implements ProjectFactory { @Override - public RelNode createProject(RelNode child, List childExprs, - List fieldNames) { + public RelNode createProject(RelNode child, + List childExprs, List fieldNames) { RelOptCluster cluster = child.getCluster(); - RelDataType rowType = - RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); + RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); + RelNode project = HiveProjectRel.create(cluster, child, + childExprs, rowType, + child.getTraitSet(), Collections. emptyList()); - return create(cluster, child, childExprs, rowType, child.getTraitSet(), - Collections.emptyList()); + return project; } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java index c3ed6c8..4738c4a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java @@ -24,7 +24,7 @@ void implement(Implementor implementor); /** Calling convention for relational operations that occur in Hive. */ - Convention CONVENTION = new Convention.Impl("HIVE", HiveRel.class); + final Convention CONVENTION = new Convention.Impl("HIVE", HiveRel.class); class Implementor { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java index df76748..bd66459 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java @@ -50,6 +50,8 @@ * @param traitSet * Traits * @param table + * Table + * @param table * HiveDB table */ public HiveTableScanRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptHiveTable table, @@ -81,10 +83,10 @@ public void implement(Implementor implementor) { @Override public double getRows() { - return table.getRowCount(); + return ((RelOptHiveTable) table).getRowCount(); } public List getColStat(List projIndxLst) { return ((RelOptHiveTable) table).getColStat(projIndxLst); } -} +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java index 063ddef..d34fe95 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java @@ -49,8 +49,7 @@ public void implement(Implementor implementor) { @Override public RelNode createSetOp(SqlKind kind, List inputs, boolean all) { if (kind != SqlKind.UNION) { - throw new IllegalStateException( - "Expected to get Set operator of type Union. Found : " + kind); + throw new IllegalStateException("Expected to get Set operator of type Union. Found : " + kind); } return new HiveUnionRel(inputs.get(0).getCluster(), inputs.get(0).getTraitSet(), inputs); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java index beb104e..6f06c6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java @@ -29,7 +29,7 @@ public class HivePartitionPrunerRule extends RelOptRule { - private HiveConf conf; + HiveConf conf; public HivePartitionPrunerRule(HiveConf conf) { super(operand(HiveFilterRel.class, operand(HiveTableScanRel.class, none()))); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java index 5aafc17..da0f7a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java @@ -110,8 +110,8 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, } } - final List aboveFilters = - filter == null ? new ArrayList() : RelOptUtil.conjunctions(filter.getCondition()); + final List aboveFilters = filter != null ? RelOptUtil + .conjunctions(filter.getCondition()) : new ArrayList(); List leftFilters = new ArrayList(); List rightFilters = new ArrayList(); @@ -155,9 +155,8 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, break; } } - if (validHiveJoinFilter) { + if (validHiveJoinFilter) continue; - } } } aboveFilters.add(exp); @@ -167,7 +166,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, /* * if all pushed filters where put back then set filterPushed to false */ - if (leftFilters.isEmpty() && rightFilters.isEmpty() + if (leftFilters.size() == 0 && rightFilters.size() == 0 && joinFilters.size() == origJoinFiltersSz) { filterPushed = false; } @@ -203,7 +202,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, // containing its new join filters (if there are any) RexNode joinFilter; - if (joinFilters.isEmpty()) { + if (joinFilters.size() == 0) { // if nothing actually got pushed and there is nothing leftover, // then this rule is a no-op if (leftFilters.isEmpty() @@ -226,7 +225,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, /** * If the filter list passed in is non-empty, creates a FilterRel on top of - * the existing RelNode; otherwise, just returns the RelNode. + * the existing RelNode; otherwise, just returns the RelNode * * @param rexBuilder * rex builder @@ -269,6 +268,7 @@ private boolean isAlwaysTrue(RexNode predicate) { } private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { + boolean refersToBothSides = false; int joinNoOfProjects = j.getRowType().getFieldCount(); BitSet filterProjs = new BitSet(joinNoOfProjects); @@ -280,11 +280,12 @@ private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { InputFinder inputFinder = new InputFinder(filterProjs); filter.accept(inputFinder); - boolean refersToBothSides = false; - if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) { + if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) refersToBothSides = true; - } return refersToBothSides; } } + +// End PushFilterPastJoinRule.java + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java index a448607..d9d94f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java @@ -42,7 +42,10 @@ * Breaks the predicate into 2 pieces. The first piece is the expressions that * only contain partition columns and can be used for Partition Pruning; the * second piece is the predicates that are left. - * + * + * @param cluster + * @param hiveTable + * @param predicate * @return a Pair of expressions, each of which maybe null. The 1st predicate * is expressions that only contain partition columns; the 2nd * predicate contains the remaining predicates. @@ -59,10 +62,10 @@ public static class ExtractPartPruningPredicate extends RexVisitorImpl { - private final RelOptHiveTable hiveTable; - private final RelDataType rType; - private final Set partCols; - private final RelOptCluster cluster; + final RelOptHiveTable hiveTable; + final RelDataType rType; + final Set partCols; + final RelOptCluster cluster; public ExtractPartPruningPredicate(RelOptCluster cluster, RelOptHiveTable hiveTable) { @@ -110,16 +113,16 @@ public RexNode visitCall(RexCall call) { } } - if (call.getOperator() == SqlStdOperatorTable.AND) { - if (args.isEmpty()) { + if (call.getOperator() != SqlStdOperatorTable.AND) { + return argsPruned ? null : call; + } else { + if (args.size() == 0) { return null; } else if (args.size() == 1) { return args.get(0); } else { return cluster.getRexBuilder().makeCall(call.getOperator(), args); } - } else { - return argsPruned ? null : call; } } @@ -127,8 +130,8 @@ public RexNode visitCall(RexCall call) { public static class ExtractRemainingPredicate extends RexVisitorImpl { - private final List pruningPredicates; - private final RelOptCluster cluster; + List pruningPredicates; + final RelOptCluster cluster; public ExtractRemainingPredicate(RelOptCluster cluster, RexNode partPruningExpr) { @@ -182,7 +185,7 @@ public RexNode visitCall(RexCall call) { } } - if (args.isEmpty()) { + if (args.size() == 0) { return null; } else if (args.size() == 1) { return args.get(0); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java index ba3ce73..83242b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java @@ -34,20 +34,19 @@ import org.eigenbase.sql.SqlKind; public class FilterSelectivityEstimator extends RexVisitorImpl { - private final RelNode childRel; - private final double childCardinality; + private final RelNode m_childRel; + private final double m_childCardinality; protected FilterSelectivityEstimator(RelNode childRel) { super(true); - this.childRel = childRel; - childCardinality = RelMetadataQuery.getRowCount(this.childRel); + m_childRel = childRel; + m_childCardinality = RelMetadataQuery.getRowCount(m_childRel); } public Double estimateSelectivity(RexNode predicate) { return predicate.accept(this); } - @Override public Double visitCall(RexCall call) { if (!deep) { return 1.0; @@ -58,7 +57,7 @@ public Double visitCall(RexCall call) { * because we have already accounted for these in * the Table row count. */ - if (isPartitionPredicate(call, childRel)) { + if (isPartitionPredicate(call, m_childRel)) { return 1.0; } @@ -66,27 +65,32 @@ public Double visitCall(RexCall call) { SqlKind op = call.getKind(); switch (op) { - case AND: + case AND: { selectivity = computeConjunctionSelectivity(call); break; + } - case OR: + case OR: { selectivity = computeDisjunctionSelectivity(call); break; + } - case NOT_EQUALS: + case NOT_EQUALS: { selectivity = computeNotEqualitySelectivity(call); + } case LESS_THAN_OR_EQUAL: case GREATER_THAN_OR_EQUAL: case LESS_THAN: - case GREATER_THAN: - selectivity = 1 / (double) 3; + case GREATER_THAN: { + selectivity = ((double) 1 / (double) 3); break; + } - case IN: - selectivity = 1 / (double) call.operands.size(); + case IN: { + selectivity = ((double) 1 / ((double) call.operands.size())); break; + } default: selectivity = computeFunctionSelectivity(call); @@ -98,15 +102,18 @@ public Double visitCall(RexCall call) { /** * NDV of "f1(x, y, z) != f2(p, q, r)" -> * "(maxNDV(x,y,z,p,q,r) - 1)/maxNDV(x,y,z,p,q,r)". + *

+ * + * @param call + * @return */ private Double computeNotEqualitySelectivity(RexCall call) { double tmpNDV = getMaxNDV(call); - if (tmpNDV > 1) { - return (tmpNDV - 1) / tmpNDV; - } else { + if (tmpNDV > 1) + return (tmpNDV - (double) 1) / tmpNDV; + else return 1.0; - } } /** @@ -114,6 +121,9 @@ private Double computeNotEqualitySelectivity(RexCall call) { *

* Note that >, >=, <, <=, = ... are considered generic functions and uses * this method to find their selectivity. + * + * @param call + * @return */ private Double computeFunctionSelectivity(RexCall call) { return 1 / getMaxNDV(call); @@ -126,42 +136,49 @@ private Double computeFunctionSelectivity(RexCall call) { *

* Note we compute m1. m2.. by applying selectivity of the disjunctive element * on the cardinality from child. + * + * @param call + * @return */ private Double computeDisjunctionSelectivity(RexCall call) { + Double tmpCardinality; + Double tmpSelectivity; double selectivity = 1; for (RexNode dje : call.getOperands()) { - Double tmpSelectivity = dje.accept(this); + tmpSelectivity = dje.accept(this); if (tmpSelectivity == null) { tmpSelectivity = 0.99; } - Double tmpCardinality = childCardinality * tmpSelectivity; + tmpCardinality = m_childCardinality * tmpSelectivity; - if (tmpCardinality > 1) { - tmpSelectivity = 1 - tmpCardinality / childCardinality; - } else { + if (tmpCardinality > 1) + tmpSelectivity = (1 - tmpCardinality / m_childCardinality); + else tmpSelectivity = 1.0; - } selectivity *= tmpSelectivity; } - if (selectivity > 1) { - return 1 - selectivity; - } else { + if (selectivity > 1) + return (1 - selectivity); + else return 1.0; - } } /** * Selectivity of conjunctive predicate -> (selectivity of conjunctive * element1) * (selectivity of conjunctive element2)... + * + * @param call + * @return */ private Double computeConjunctionSelectivity(RexCall call) { + Double tmpSelectivity; double selectivity = 1; for (RexNode cje : call.getOperands()) { - Double tmpSelectivity = cje.accept(this); + tmpSelectivity = cje.accept(this); if (tmpSelectivity != null) { selectivity *= tmpSelectivity; } @@ -171,20 +188,21 @@ private Double computeConjunctionSelectivity(RexCall call) { } private Double getMaxNDV(RexCall call) { + double tmpNDV; double maxNDV = 1.0; + InputReferencedVisitor irv; for (RexNode op : call.getOperands()) { - double tmpNDV; if (op instanceof RexInputRef) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(childRel, + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, ((RexInputRef) op).getIndex()); if (tmpNDV > maxNDV) maxNDV = tmpNDV; } else { - InputReferencedVisitor irv = new InputReferencedVisitor(); + irv = new InputReferencedVisitor(); irv.apply(op); for (Integer childProjIndx : irv.inputPosReferenced) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(childRel, childProjIndx); + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, childProjIndx); if (tmpNDV > maxNDV) maxNDV = tmpNDV; } @@ -195,13 +213,14 @@ private Double getMaxNDV(RexCall call) { } private boolean isPartitionPredicate(RexNode expr, RelNode r) { - if (r instanceof ProjectRelBase) { + if ( r instanceof ProjectRelBase ) { expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r); return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild()); - } else if (r instanceof FilterRelBase) { + } else if ( r instanceof FilterRelBase ) { isPartitionPredicate(expr, ((ProjectRelBase) r).getChild()); - } else if (r instanceof HiveTableScanRel) { - RelOptHiveTable table = (RelOptHiveTable) r.getTable(); + } else if ( r instanceof HiveTableScanRel ) { + RelOptHiveTable table = (RelOptHiveTable) + ((HiveTableScanRel)r).getTable(); BitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java index f9578f8..d87c808 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java @@ -52,7 +52,7 @@ public class HiveRelMdRowCount extends RelMdRowCount { - private static final Log LOG = LogFactory.getLog(HiveRelMdRowCount.class.getName()); + protected static final Log LOG = LogFactory.getLog(HiveRelMdRowCount.class.getName()); public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider @@ -65,7 +65,7 @@ protected HiveRelMdRowCount() { public Double getRowCount(JoinRelBase join) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(join); if (pkfk != null) { - double selectivity = pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor; + double selectivity = (pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor); selectivity = Math.min(1.0, selectivity); if (LOG.isDebugEnabled()) { LOG.debug("Identified Primary - Foreign Key relation:"); @@ -77,11 +77,10 @@ public Double getRowCount(JoinRelBase join) { return join.getRows(); } - @Override public Double getRowCount(SemiJoinRel rel) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(rel); if (pkfk != null) { - double selectivity = pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor; + double selectivity = (pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor); selectivity = Math.min(1.0, selectivity); if (LOG.isDebugEnabled()) { LOG.debug("Identified Primary - Foreign Key relation:"); @@ -127,7 +126,7 @@ public String toString() { static class FKSideInfo { public final double rowCount; public final double distinctCount; - FKSideInfo(double rowCount, double distinctCount) { + public FKSideInfo(double rowCount, double distinctCount) { this.rowCount = rowCount; this.distinctCount = distinctCount; } @@ -139,14 +138,13 @@ public String toString() { static class PKSideInfo extends FKSideInfo { public final double selectivity; - PKSideInfo(double rowCount, double distinctCount, double selectivity) { + public PKSideInfo(double rowCount, double distinctCount, double selectivity) { super(rowCount, distinctCount); this.selectivity = selectivity; } public String toString() { - return String.format("PKInfo(rowCount=%.2f,ndv=%.2f,selectivity=%.2f)", rowCount, - distinctCount, selectivity); + return String.format("PKInfo(rowCount=%.2f,ndv=%.2f,selectivity=%.2f)", rowCount, distinctCount,selectivity); } } @@ -229,20 +227,21 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { double rightRowCount = RelMetadataQuery.getRowCount(right) * RelMetadataQuery.getSelectivity(right, rightPred); - if (leftIsKey && rightIsKey && rightRowCount < leftRowCount) { - leftIsKey = false; + if (leftIsKey && rightIsKey) { + if (rightRowCount < leftRowCount) { + leftIsKey = false; + } } int pkSide = leftIsKey ? 0 : rightIsKey ? 1 : -1; - boolean isPKSideSimpleTree = pkSide != -1 && - IsSimpleTreeOnJoinKey.check(pkSide == 0 ? left : right, - pkSide == 0 ? leftColIdx : rightColIdx); + boolean isPKSideSimpleTree = pkSide != -1 ? + IsSimpleTreeOnJoinKey.check( + pkSide == 0 ? left : right, + pkSide == 0 ? leftColIdx : rightColIdx) : false; - double leftNDV = - isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(left, lBitSet, leftPred) : -1; - double rightNDV = - isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(right, rBitSet, rightPred) : -1; + double leftNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(left, lBitSet, leftPred) : -1; + double rightNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(right, rBitSet, rightPred) : -1; /* * If the ndv of the PK - FK side don't match, and the PK side is a filter @@ -266,9 +265,9 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { * d_date column we can apply the scaling factor. */ double ndvScalingFactor = 1.0; - if (isPKSideSimpleTree) { - ndvScalingFactor = pkSide == 0 ? leftNDV / rightNDV : rightNDV / leftNDV; - } + if ( isPKSideSimpleTree ) { + ndvScalingFactor = pkSide == 0 ? leftNDV/rightNDV : rightNDV / leftNDV; + } if (pkSide == 0) { FKSideInfo fkInfo = new FKSideInfo(rightRowCount, @@ -328,6 +327,9 @@ private static boolean isKey(BitSet c, RelNode rel) { RexNode joinCond = joinFilters.get(0); + int leftColIdx; + int rightColIdx; + if (!(joinCond instanceof RexCall)) { return null; } @@ -339,7 +341,7 @@ private static boolean isKey(BitSet c, RelNode rel) { BitSet leftCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(0)); BitSet rightCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(1)); - if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1) { + if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1 ) { return null; } @@ -358,16 +360,16 @@ private static boolean isKey(BitSet c, RelNode rel) { rightCols = t; } - int leftColIdx = leftCols.nextSetBit(0) - nSysFields; - int rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft); + leftColIdx = leftCols.nextSetBit(0) - nSysFields; + rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft); return new Pair(leftColIdx, rightColIdx); } private static class IsSimpleTreeOnJoinKey extends RelVisitor { - private int joinKey; - private boolean simpleTree; + int joinKey; + boolean simpleTree; static boolean check(RelNode r, int joinKey) { IsSimpleTreeOnJoinKey v = new IsSimpleTreeOnJoinKey(joinKey); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java index dd89166..5d9b145 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java @@ -46,6 +46,7 @@ new HiveRelMdSelectivity()); protected HiveRelMdSelectivity() { + super(); } public Double getSelectivity(HiveTableScanRel t, RexNode predicate) { @@ -58,18 +59,20 @@ public Double getSelectivity(HiveTableScanRel t, RexNode predicate) { } public Double getSelectivity(HiveJoinRel j, RexNode predicate) { - if (j.getJoinType() == JoinRelType.INNER) { + if (j.getJoinType().equals(JoinRelType.INNER)) { return computeInnerJoinSelectivity(j, predicate); } return 1.0; } private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { + double ndvCrossProduct = 1; RexNode combinedPredicate = getCombinedPredicateForJoin(j, predicate); JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, combinedPredicate); ImmutableMap.Builder colStatMapBuilder = ImmutableMap .builder(); + ImmutableMap colStatMap; int rightOffSet = j.getLeft().getRowType().getFieldCount(); // 1. Update Col Stats Map with col stats for columns from left side of @@ -85,27 +88,25 @@ private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { colStatMapBuilder.put(rjk + rightOffSet, HiveRelMdDistinctRowCount.getDistinctRowCount(j.getRight(), rjk)); } - ImmutableMap colStatMap = colStatMapBuilder.build(); + colStatMap = colStatMapBuilder.build(); // 3. Walk through the Join Condition Building NDV for selectivity // NDV of the join can not exceed the cardinality of cross join. List peLst = jpi.getEquiJoinPredicateElements(); int noOfPE = peLst.size(); - double ndvCrossProduct = 1; if (noOfPE > 0) { ndvCrossProduct = exponentialBackoff(peLst, colStatMap); - if (j.isLeftSemiJoin()) { - ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()), ndvCrossProduct); - } else { - ndvCrossProduct = Math.min( - RelMetadataQuery.getRowCount(j.getLeft()) * RelMetadataQuery.getRowCount(j.getRight()), - ndvCrossProduct); - } - } + if (j.isLeftSemiJoin()) + ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()), + ndvCrossProduct); + else + ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()) + * RelMetadataQuery.getRowCount(j.getRight()), ndvCrossProduct); + } // 4. Join Selectivity = 1/NDV - return 1 / ndvCrossProduct; + return (1 / ndvCrossProduct); } // 3.2 if conjunctive predicate elements are more than one, then walk @@ -131,10 +132,11 @@ protected double logSmoothing(List peLst, ImmutableMap 1) { double maxNDVSoFar = ndvCrossProduct; + double ndvToBeSmoothed; + double tmpNDV; for (int i = 1; i < noOfPE; i++) { - double tmpNDV = getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap); - double ndvToBeSmoothed; + tmpNDV = getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap); if (tmpNDV > maxNDVSoFar) { ndvToBeSmoothed = maxNDVSoFar; maxNDVSoFar = tmpNDV; @@ -143,11 +145,10 @@ protected double logSmoothing(List peLst, ImmutableMap 3) { + if (ndvToBeSmoothed > 3) ndvCrossProduct *= Math.log(ndvToBeSmoothed); - } else { + else ndvCrossProduct *= ndvToBeSmoothed; - } } } return ndvCrossProduct; @@ -161,8 +162,8 @@ protected double exponentialBackoff(List peLst, ImmutableMap colStatMap) { int noOfPE = peLst.size(); List ndvs = new ArrayList(noOfPE); - for (JoinLeafPredicateInfo aPeLst : peLst) { - ndvs.add(getMaxNDVForJoinSelectivity(aPeLst, colStatMap)); + for (int i = 0; i < noOfPE; i++) { + ndvs.add(getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap)); } Collections.sort(ndvs); Collections.reverse(ndvs); @@ -191,9 +192,12 @@ private RexNode getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPre /** * Compute Max NDV to determine Join Selectivity. - * + * + * @param jlpi * @param colStatMap * Immutable Map of Projection Index (in Join Schema) to Column Stat + * @param rightProjOffSet + * @return */ private static Double getMaxNDVForJoinSelectivity(JoinLeafPredicateInfo jlpi, ImmutableMap colStatMap) { @@ -209,15 +213,16 @@ private static Double getMaxNDVForJoinSelectivity(JoinLeafPredicateInfo jlpi, private static Double getMaxNDVFromProjections(Map colStatMap, Set projectionSet, Double defaultMaxNDV) { + Double colNDV = null; Double maxNDVSoFar = defaultMaxNDV; for (Integer projIndx : projectionSet) { - Double colNDV = colStatMap.get(projIndx); + colNDV = colStatMap.get(projIndx); if (colNDV > maxNDVSoFar) maxNDVSoFar = colNDV; } return maxNDVSoFar; } - + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java index 7c22b39..06ff584 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java @@ -51,7 +51,7 @@ /* * Infer Uniquenes if: - rowCount(col) = ndv(col) - TBD for numerics: max(col) * - min(col) = rowCount(col) - * + * * Why are we intercepting ProjectRelbase and not TableScan? Because if we * have a method for TableScan, it will not know which columns to check for. * Inferring Uniqueness for all columns is very expensive right now. The flip @@ -94,14 +94,14 @@ if (cStat.getCountDistint() >= numRows) { isKey = true; } - if (!isKey && cStat.getRange() != null && - cStat.getRange().maxValue != null && + if ( !isKey && cStat.getRange() != null && + cStat.getRange().maxValue != null && cStat.getRange().minValue != null) { - double r = - cStat.getRange().maxValue.doubleValue() - cStat.getRange().minValue.doubleValue() + 1; - isKey = numRows == r; + double r = cStat.getRange().maxValue.doubleValue() - + cStat.getRange().minValue.doubleValue() + 1; + isKey = (numRows == r); } - if (isKey) { + if ( isKey ) { BitSet key = new BitSet(); key.set(posMap.get(colStatsPos)); keys.add(key); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java index a02abc3..3762988 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java @@ -45,17 +45,19 @@ static ASTNode createAST(int tokenType, String text) { } static ASTNode destNode() { - return construct(HiveParser.TOK_DESTINATION, "TOK_DESTINATION") + return ASTBuilder + .construct(HiveParser.TOK_DESTINATION, "TOK_DESTINATION") .add( - construct(HiveParser.TOK_DIR, "TOK_DIR").add(HiveParser.TOK_TMP_FILE, + ASTBuilder.construct(HiveParser.TOK_DIR, "TOK_DIR").add(HiveParser.TOK_TMP_FILE, "TOK_TMP_FILE")).node(); } static ASTNode table(TableAccessRelBase scan) { RelOptHiveTable hTbl = (RelOptHiveTable) scan.getTable(); - ASTBuilder b = construct(HiveParser.TOK_TABREF, "TOK_TABREF") + ASTBuilder b = ASTBuilder + .construct(HiveParser.TOK_TABREF, "TOK_TABREF") .add( - construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") + ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()) .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName())) .add(HiveParser.Identifier, hTbl.getName()); @@ -69,19 +71,19 @@ static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode c switch (joinType) { case INNER: if (semiJoin) { - b = construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); + b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); } else { - b = construct(HiveParser.TOK_JOIN, "TOK_JOIN"); + b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN"); } break; case LEFT: - b = construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN"); + b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN"); break; case RIGHT: - b = construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN"); + b = ASTBuilder.construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN"); break; case FULL: - b = construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN"); + b = ASTBuilder.construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN"); break; } @@ -90,45 +92,46 @@ static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode c } static ASTNode subQuery(ASTNode qry, String alias) { - return construct(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY").add(qry) + return ASTBuilder.construct(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY").add(qry) .add(HiveParser.Identifier, alias).node(); } static ASTNode qualifiedName(String tableName, String colName) { - ASTBuilder b = construct(HiveParser.DOT, ".") + ASTBuilder b = ASTBuilder + .construct(HiveParser.DOT, ".") .add( - construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( + ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( HiveParser.Identifier, tableName)).add(HiveParser.Identifier, colName); return b.node(); } static ASTNode unqualifiedName(String colName) { - ASTBuilder b = construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( + ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( HiveParser.Identifier, colName); return b.node(); } static ASTNode where(ASTNode cond) { - return construct(HiveParser.TOK_WHERE, "TOK_WHERE").add(cond).node(); + return ASTBuilder.construct(HiveParser.TOK_WHERE, "TOK_WHERE").add(cond).node(); } static ASTNode having(ASTNode cond) { - return construct(HiveParser.TOK_HAVING, "TOK_HAVING").add(cond).node(); + return ASTBuilder.construct(HiveParser.TOK_HAVING, "TOK_HAVING").add(cond).node(); } static ASTNode limit(Object value) { - return construct(HiveParser.TOK_LIMIT, "TOK_LIMIT") + return ASTBuilder.construct(HiveParser.TOK_LIMIT, "TOK_LIMIT") .add(HiveParser.Number, value.toString()).node(); } static ASTNode selectExpr(ASTNode expr, String alias) { - return construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr) + return ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr) .add(HiveParser.Identifier, alias).node(); } static ASTNode literal(RexLiteral literal) { Object val = null; - int type; + int type = 0; SqlTypeName sqlType = literal.getType().getSqlTypeName(); switch (sqlType) { @@ -151,11 +154,11 @@ static ASTNode literal(RexLiteral literal) { type = HiveParser.BigintLiteral; break; case DOUBLE: - val = literal.getValue3() + "D"; + val = literal.getValue3()+"D"; type = HiveParser.Number; break; case DECIMAL: - val = literal.getValue3() + "BD"; + val = literal.getValue3()+"BD"; type = HiveParser.DecimalLiteral; break; case FLOAT: @@ -167,11 +170,11 @@ static ASTNode literal(RexLiteral literal) { case CHAR: val = literal.getValue3(); type = HiveParser.StringLiteral; - val = "'" + val + "'"; + val = "'" + String.valueOf(val) + "'"; break; case BOOLEAN: val = literal.getValue3(); - type = (Boolean) val ? HiveParser.KW_TRUE : HiveParser.KW_FALSE; + type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE : HiveParser.KW_FALSE; break; case DATE: { val = literal.getValue(); @@ -179,15 +182,16 @@ static ASTNode literal(RexLiteral literal) { DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); val = df.format(((Calendar) val).getTime()); val = "'" + val + "'"; - } + } break; case TIME: - case TIMESTAMP: + case TIMESTAMP: { val = literal.getValue(); type = HiveParser.TOK_TIMESTAMP; DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); val = df.format(((Calendar) val).getTime()); val = "'" + val + "'"; + } break; case NULL: type = HiveParser.TOK_NULL; @@ -200,7 +204,7 @@ static ASTNode literal(RexLiteral literal) { return (ASTNode) ParseDriver.adaptor.create(type, String.valueOf(val)); } - private ASTNode curr; + ASTNode curr; ASTNode node() { return curr; @@ -222,4 +226,4 @@ ASTBuilder add(ASTNode n) { } return this; } -} +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java index 8e628d5..77427e8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java @@ -63,14 +63,16 @@ public class ASTConverter { - private final RelNode root; - private final HiveAST hiveAST; - private RelNode from; - private FilterRelBase where; - private AggregateRelBase groupBy; - private FilterRelBase having; - private ProjectRelBase select; - private SortRel order; + RelNode root; + HiveAST hiveAST; + RelNode from; + FilterRelBase where; + AggregateRelBase groupBy; + FilterRelBase having; + ProjectRelBase select; + SortRel order; + + Schema schema; ASTConverter(RelNode root) { this.root = root; @@ -85,9 +87,8 @@ public static ASTNode convert(final RelNode relNode, List resultSch if (root instanceof SortRel) { sortrel = (SortRel) root; root = sortrel.getChild(); - if (!(root instanceof ProjectRelBase)) { + if (!(root instanceof ProjectRelBase)) throw new RuntimeException("Child of root sort node is not a project"); - } } ASTConverter c = new ASTConverter(root); @@ -104,7 +105,7 @@ public ASTNode convert(SortRel sortrel) { * 2. convert from node. */ QueryBlockInfo qb = convertSource(from); - Schema schema = qb.schema; + schema = qb.schema; hiveAST.from = ASTBuilder.construct(HiveParser.TOK_FROM, "TOK_FROM").add(qb.ast).node(); /* @@ -126,9 +127,8 @@ public ASTNode convert(SortRel sortrel) { b.add(iRef.accept(new RexVisitor(schema))); } - if (!groupBy.getGroupSet().isEmpty()) { + if (!groupBy.getGroupSet().isEmpty()) hiveAST.groupBy = b.node(); - } schema = new Schema(schema, groupBy); } @@ -146,7 +146,7 @@ public ASTNode convert(SortRel sortrel) { ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT"); if (select.getChildExps().isEmpty()) { - RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.ONE); + RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(new BigDecimal(1)); ASTNode selectExpr = ASTBuilder.selectExpr(ASTBuilder.literal(r), "1"); b.add(selectExpr); } else { @@ -217,27 +217,26 @@ private QueryBlockInfo convertSource(RelNode r) { ASTNode cond = join.getCondition().accept(new RexVisitor(s)); boolean semiJoin = join instanceof SemiJoinRel; ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin); - if (semiJoin) { + if (semiJoin) s = left.schema; - } } else if (r instanceof UnionRelBase) { - RelNode leftInput = r.getInput(0); - RelNode rightInput = r.getInput(1); + RelNode leftInput = ((UnionRelBase) r).getInput(0); + RelNode rightInput = ((UnionRelBase) r).getInput(1); ASTConverter leftConv = new ASTConverter(leftInput); ASTConverter rightConv = new ASTConverter(rightInput); - ASTNode leftAST = leftConv.convert(null); - ASTNode rightAST = rightConv.convert(null); + ASTNode leftAST = leftConv.convert((SortRel) null); + ASTNode rightAST = rightConv.convert((SortRel) null); ASTNode unionAST = getUnionAllAST(leftAST, rightAST); - String sqAlias = nextAlias(); + String sqAlias = ASTConverter.nextAlias(); ast = ASTBuilder.subQuery(unionAST, sqAlias); s = new Schema((UnionRelBase) r, sqAlias); } else { ASTConverter src = new ASTConverter(r); ASTNode srcAST = src.convert(order); - String sqAlias = nextAlias(); + String sqAlias = ASTConverter.nextAlias(); s = src.getRowSchema(sqAlias); ast = ASTBuilder.subQuery(srcAST, sqAlias); } @@ -248,19 +247,18 @@ private QueryBlockInfo convertSource(RelNode r) { public void handle(FilterRelBase filter) { RelNode child = filter.getChild(); - if (child instanceof AggregateRelBase && - !((AggregateRelBase) child).getGroupSet().isEmpty()) { - having = filter; + if (child instanceof AggregateRelBase && !((AggregateRelBase) child).getGroupSet().isEmpty()) { + ASTConverter.this.having = filter; } else { - where = filter; + ASTConverter.this.where = filter; } } public void handle(ProjectRelBase project) { - if (select == null) { - select = project; + if (ASTConverter.this.select == null) { + ASTConverter.this.select = project; } else { - from = project; + ASTConverter.this.from = project; } } @@ -268,24 +266,24 @@ public void handle(ProjectRelBase project) { public void visit(RelNode node, int ordinal, RelNode parent) { if (node instanceof TableAccessRelBase) { - from = node; + ASTConverter.this.from = node; } else if (node instanceof FilterRelBase) { handle((FilterRelBase) node); } else if (node instanceof ProjectRelBase) { handle((ProjectRelBase) node); } else if (node instanceof JoinRelBase) { - from = node; + ASTConverter.this.from = node; } else if (node instanceof UnionRelBase) { - from = node; + ASTConverter.this.from = node; } else if (node instanceof AggregateRelBase) { - groupBy = (AggregateRelBase) node; + ASTConverter.this.groupBy = (AggregateRelBase) node; } else if (node instanceof SortRel) { - order = (SortRel) node; + ASTConverter.this.order = (SortRel) node; } /* * once the source node is reached; stop traversal for this QB */ - if (from == null) { + if (ASTConverter.this.from == null) { node.childrenAccept(this); } } @@ -322,6 +320,7 @@ public ASTNode visitLiteral(RexLiteral literal) { } private ASTNode getPSpecAST(RexWindow window) { + ASTNode pSpecAst = null; ASTNode dByAst = null; if (window.partitionKeys != null && !window.partitionKeys.isEmpty()) { @@ -345,31 +344,27 @@ private ASTNode getPSpecAST(RexWindow window) { } } - ASTNode pSpecAst = null; if (dByAst != null || oByAst != null) { pSpecAst = ASTBuilder.createAST(HiveParser.TOK_PARTITIONINGSPEC, "TOK_PARTITIONINGSPEC"); - if (dByAst != null) { + if (dByAst != null) pSpecAst.addChild(dByAst); - } - if (oByAst != null) { + if (oByAst != null) pSpecAst.addChild(oByAst); - } } return pSpecAst; } private ASTNode getWindowBound(RexWindowBound wb) { - ASTNode wbAST; + ASTNode wbAST = null; if (wb.isCurrentRow()) { wbAST = ASTBuilder.createAST(HiveParser.KW_CURRENT, "CURRENT"); } else { - if (wb.isPreceding()) { + if (wb.isPreceding()) wbAST = ASTBuilder.createAST(HiveParser.KW_PRECEDING, "PRECEDING"); - } else { + else wbAST = ASTBuilder.createAST(HiveParser.KW_FOLLOWING, "FOLLOWING"); - } if (wb.isUnbounded()) { wbAST.addChild(ASTBuilder.createAST(HiveParser.KW_UNBOUNDED, "UNBOUNDED")); } else { @@ -382,6 +377,7 @@ private ASTNode getWindowBound(RexWindowBound wb) { } private ASTNode getWindowRangeAST(RexWindow window) { + ASTNode wRangeAst = null; ASTNode startAST = null; RexWindowBound ub = window.getUpperBound(); @@ -395,20 +391,16 @@ private ASTNode getWindowRangeAST(RexWindow window) { endAST = getWindowBound(lb); } - ASTNode wRangeAst = null; if (startAST != null || endAST != null) { // NOTE: in Hive AST Rows->Range(Physical) & Range -> Values (logical) - if (window.isRows()) { + if (window.isRows()) wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWRANGE, "TOK_WINDOWRANGE"); - } else { + else wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWVALUES, "TOK_WINDOWVALUES"); - } - if (startAST != null) { + if (startAST != null) wRangeAst.addChild(startAST); - } - if (endAST != null) { + if (endAST != null) wRangeAst.addChild(endAST); - } } return wRangeAst; @@ -431,12 +423,10 @@ public ASTNode visitOver(RexOver over) { final RexWindow window = over.getWindow(); final ASTNode wPSpecAst = getPSpecAST(window); final ASTNode wRangeAst = getWindowRangeAST(window); - if (wPSpecAst != null) { + if (wPSpecAst != null) wSpec.addChild(wPSpecAst); - } - if (wRangeAst != null) { + if (wRangeAst != null) wSpec.addChild(wRangeAst); - } return wUDAFAst; } @@ -453,9 +443,8 @@ public ASTNode visitCall(RexCall call) { HiveToken ht = TypeConverter.hiveToken(call.getType()); ASTBuilder astBldr = ASTBuilder.construct(ht.type, ht.text); if (ht.args != null) { - for (String castArg : ht.args) { + for (String castArg : ht.args) astBldr.add(HiveParser.Identifier, castArg); - } } astNodeLst.add(astBldr.node()); } @@ -464,20 +453,19 @@ public ASTNode visitCall(RexCall call) { astNodeLst.add(operand.accept(this)); } - if (isFlat(call)) { + if (isFlat(call)) return SqlFunctionConverter.buildAST(op, astNodeLst, 0); - } else { + else return SqlFunctionConverter.buildAST(op, astNodeLst); - } } } static class QueryBlockInfo { - - Schema schema; + Schema schema; ASTNode ast; - QueryBlockInfo(Schema schema, ASTNode ast) { + public QueryBlockInfo(Schema schema, ASTNode ast) { + super(); this.schema = schema; this.ast = ast; } @@ -509,6 +497,7 @@ public ASTNode visitCall(RexCall call) { } } + @SuppressWarnings("unchecked") Schema(Schema left, Schema right) { for (ColumnInfo cI : Iterables.concat(left, right)) { add(cI); @@ -541,11 +530,12 @@ public ASTNode visitCall(RexCall call) { * 1. ProjectRel will always be child of SortRel.
* 2. In Optiq every projection in ProjectRelBase is uniquely named * (unambigous) without using table qualifier (table name).
- * + * * @param order * Hive Sort Rel Node + * @return Schema */ - Schema(HiveSortRel order) { + public Schema(HiveSortRel order) { ProjectRelBase select = (ProjectRelBase) order.getChild(); for (String projName : select.getRowType().getFieldNames()) { add(new ColumnInfo(null, projName)); @@ -562,27 +552,29 @@ public ASTNode visitCall(RexCall call) { ASTNode agg; ColumnInfo(String table, String column) { + super(); this.table = table; this.column = column; } ColumnInfo(String table, ASTNode agg) { + super(); this.table = table; this.agg = agg; } ColumnInfo(String alias, ColumnInfo srcCol) { - table = alias; - column = srcCol.column; - agg = srcCol.agg; + this.table = alias; + this.column = srcCol.column; + this.agg = srcCol.agg; } } static String nextAlias() { - return String.format("$hdt$_%d", DERIVED_TABLE_COUNTER.getAndIncrement()); + return String.format("$hdt$_%d", derivedTableCounter.getAndIncrement()); } - private static final AtomicLong DERIVED_TABLE_COUNTER = new AtomicLong(0); + private static AtomicLong derivedTableCounter = new AtomicLong(0); static class HiveAST { @@ -607,8 +599,10 @@ public ASTNode getAST() { public ASTNode getUnionAllAST(ASTNode leftAST, ASTNode rightAST) { - return ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST) + ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST) .add(rightAST).node(); + + return unionTokAST; } public static boolean isFlat(RexCall call) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java index b3abd40..e324b9c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/DerivedTableInjector.java @@ -44,14 +44,9 @@ import org.eigenbase.relopt.volcano.RelSubset; import org.eigenbase.rex.RexNode; -public final class DerivedTableInjector { +public class DerivedTableInjector { - private DerivedTableInjector() { - throw new UnsupportedOperationException("Can't initialize class"); - } - - public static RelNode convertOpTree(RelNode rel, List resultSchema) - throws OptiqSemanticException { + public static RelNode convertOpTree(RelNode rel, List resultSchema) throws OptiqSemanticException { RelNode newTopNode = rel; if (!(newTopNode instanceof ProjectRelBase) && !(newTopNode instanceof SortRel)) { @@ -82,9 +77,8 @@ private static void convertOpTree(RelNode rel, RelNode parent) { throw new RuntimeException("Found RelSubset"); } else if (rel instanceof SetOpRel) { // TODO: Handle more than 2 inputs for setop - if (!validSetopParent(rel, parent)) { + if (!validSetopParent(rel, parent)) introduceDerivedTable(rel, parent); - } SetOpRel setopRel = (SetOpRel) rel; for (RelNode inputRel : setopRel.getInputs()) { @@ -104,8 +98,10 @@ private static void convertOpTree(RelNode rel, RelNode parent) { if (!validSortChild((HiveSortRel) rel)) { introduceDerivedTable(((HiveSortRel) rel).getChild(), rel); } - } else if (rel instanceof HiveAggregateRel && !validGBParent(rel, parent)) { - introduceDerivedTable(rel, parent); + } else if (rel instanceof HiveAggregateRel) { + if (!validGBParent(rel, parent)) { + introduceDerivedTable(rel, parent); + } } } else if (rel instanceof TableAccessRelBase) { @@ -164,8 +160,10 @@ private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, private static RelNode introduceDerivedTable(final RelNode rel) { List projectList = HiveOptiqUtil.getProjsFromBelowAsInputRef(rel); - return HiveProjectRel.create(rel.getCluster(), rel, projectList, + HiveProjectRel select = HiveProjectRel.create(rel.getCluster(), rel, projectList, rel.getRowType(), rel.getCollationList()); + + return select; } private static void introduceDerivedTable(final RelNode rel, RelNode parent) { @@ -207,7 +205,7 @@ private static boolean validJoinParent(RelNode joinNode, RelNode parent) { private static boolean validFilterParent(RelNode filterNode, RelNode parent) { boolean validParent = true; - // TODO: Verify GB having is not a seperate filter (if so we shouldn't + // TOODO: Verify GB having is not a seperate filter (if so we shouldn't // introduce derived table) if (parent instanceof FilterRelBase || parent instanceof JoinRelBase || parent instanceof SetOpRel) { @@ -220,7 +218,7 @@ private static boolean validFilterParent(RelNode filterNode, RelNode parent) { private static boolean validGBParent(RelNode gbNode, RelNode parent) { boolean validParent = true; - // TODO: Verify GB having is not a seperate filter (if so we shouldn't + // TOODO: Verify GB having is not a seperate filter (if so we shouldn't // introduce derived table) if (parent instanceof JoinRelBase || parent instanceof SetOpRel || parent instanceof AggregateRelBase diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java index 5548151..e6e491f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java @@ -43,9 +43,9 @@ */ public class ExprNodeConverter extends RexVisitorImpl { - private final RelDataType rType; - private final String tabAlias; - private final boolean partitioningExpr; + RelDataType rType; + String tabAlias; + boolean partitioningExpr; public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitioningExpr) { super(true); @@ -56,7 +56,7 @@ public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitionin * on whether a column is a partition column or not, whether a column is a * virtual column or not. */ - assert partitioningExpr; + assert partitioningExpr == true; this.tabAlias = tabAlias; this.rType = rType; this.partitioningExpr = partitioningExpr; @@ -71,6 +71,7 @@ public ExprNodeDesc visitInputRef(RexInputRef inputRef) { @Override public ExprNodeDesc visitCall(RexCall call) { + ExprNodeGenericFuncDesc gfDesc = null; if (!deep) { return null; @@ -84,9 +85,8 @@ public ExprNodeDesc visitCall(RexCall call) { // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the // exprnode - ExprNodeGenericFuncDesc gfDesc; if (ASTConverter.isFlat(call)) { - List tmpExprArgs = new ArrayList(); + ArrayList tmpExprArgs = new ArrayList(); tmpExprArgs.addAll(args.subList(0, 2)); gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()), SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType()), tmpExprArgs); @@ -111,29 +111,29 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { switch (literal.getType().getSqlTypeName()) { case BOOLEAN: - return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, - RexLiteral.booleanValue(literal)); + return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.valueOf(RexLiteral + .booleanValue(literal))); case TINYINT: - return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, - ((Number) literal.getValue3()).byteValue()); + return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, Byte.valueOf(((Number) literal + .getValue3()).byteValue())); case SMALLINT: return new ExprNodeConstantDesc(TypeInfoFactory.shortTypeInfo, - ((Number) literal.getValue3()).shortValue()); + Short.valueOf(((Number) literal.getValue3()).shortValue())); case INTEGER: return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, - ((Number) literal.getValue3()).intValue()); + Integer.valueOf(((Number) literal.getValue3()).intValue())); case BIGINT: - return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, - ((Number) literal.getValue3()).longValue()); + return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, Long.valueOf(((Number) literal + .getValue3()).longValue())); case FLOAT: return new ExprNodeConstantDesc(TypeInfoFactory.floatTypeInfo, - ((Number) literal.getValue3()).floatValue()); + Float.valueOf(((Number) literal.getValue3()).floatValue())); case DOUBLE: return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo, - ((Number) literal.getValue3()).doubleValue()); + Double.valueOf(((Number) literal.getValue3()).doubleValue())); case DATE: return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, - new Date(((Calendar) literal.getValue()).getTimeInMillis())); + new Date(((Calendar)literal.getValue()).getTimeInMillis())); case TIMESTAMP: return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, literal.getValue3()); case BINARY: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java index f15b05d..406c18e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java @@ -48,8 +48,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; /** - * JoinCondTypeCheckProcFactory is used by Optiq planner(CBO) to generate Join Conditions from - * Join Condition AST. + * JoinCondTypeCheckProcFactory is used by Optiq planner(CBO) to generate Join Conditions from Join Condition AST. * Reasons for sub class: * 1. Additional restrictions on what is supported in Join Conditions * 2. Column handling is different @@ -91,13 +90,14 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } - assert expr.getChildCount() == 1; + assert (expr.getChildCount() == 1); String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); - boolean qualifiedAccess = parent != null && parent.getType() == HiveParser.DOT; + boolean qualifiedAccess = (parent != null && parent.getType() == HiveParser.DOT); + ColumnInfo colInfo = null; if (!qualifiedAccess) { - ColumnInfo colInfo = getColInfo(ctx, null, tableOrCol, expr); + colInfo = getColInfo(ctx, null, tableOrCol, expr); // It's a column. return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol()); @@ -112,24 +112,24 @@ private static boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNo throws SemanticException { int tblAliasCnt = 0; for (RowResolver rr : ctx.getInputRRList()) { - if (rr.hasTableAlias(tabName)) { + if (rr.hasTableAlias(tabName)) tblAliasCnt++; - } } if (tblAliasCnt > 1) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); } - return tblAliasCnt == 1; + return (tblAliasCnt == 1) ? true : false; } private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { + ColumnInfo tmp; ColumnInfo cInfoToRet = null; for (RowResolver rr : ctx.getInputRRList()) { - ColumnInfo tmp = rr.get(tabName, colAlias); + tmp = rr.get(tabName, colAlias); if (tmp != null) { if (cInfoToRet != null) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); @@ -144,7 +144,7 @@ private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, Strin /** * Factory method to get ColumnExprProcessor. - * + * * @return ColumnExprProcessor. */ @Override @@ -187,10 +187,11 @@ protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode ex private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { + ColumnInfo tmp; ColumnInfo cInfoToRet = null; for (RowResolver rr : ctx.getInputRRList()) { - ColumnInfo tmp = rr.get(tabName, colAlias); + tmp = rr.get(tabName, colAlias); if (tmp != null) { if (cInfoToRet != null) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); @@ -224,12 +225,12 @@ protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, F } else if (genericUDF instanceof GenericUDFBaseCompare) { // Comparisons of non literals LHS/RHS can not refer to inputs from // both sides - if (children.size() == 2 && - !(children.get(0) instanceof ExprNodeConstantDesc) && - !(children.get(1) instanceof ExprNodeConstantDesc) && - comparisonUDFargsRefersToBothInput((GenericUDFBaseCompare) genericUDF, children, + if (children.size() == 2 && !(children.get(0) instanceof ExprNodeConstantDesc) + && !(children.get(1) instanceof ExprNodeConstantDesc)) { + if (comparisonUDFargsRefersToBothInput((GenericUDFBaseCompare) genericUDF, children, jCtx.getInputRRList())) { - throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); + throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); + } } } } @@ -237,6 +238,7 @@ protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, F private static boolean genericUDFargsRefersToBothInput(GenericUDF udf, List children, List inputRRList) { + boolean argsRefersToBothInput = false; Map hasCodeToColDescMap = new HashMap(); for (ExprNodeDesc child : children) { @@ -244,16 +246,15 @@ private static boolean genericUDFargsRefersToBothInput(GenericUDF udf, } Set inputRef = getInputRef(hasCodeToColDescMap.values(), inputRRList); - boolean argsRefersToBothInput = false; - if (inputRef.size() > 1) { + if (inputRef.size() > 1) argsRefersToBothInput = true; - } return argsRefersToBothInput; } private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare comparisonUDF, List children, List inputRRList) { + boolean argsRefersToBothInput = false; Map lhsHashCodeToColDescMap = new HashMap(); Map rhsHashCodeToColDescMap = new HashMap(); @@ -262,39 +263,39 @@ private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare Set lhsInputRef = getInputRef(lhsHashCodeToColDescMap.values(), inputRRList); Set rhsInputRef = getInputRef(rhsHashCodeToColDescMap.values(), inputRRList); - boolean argsRefersToBothInput = false; - if (lhsInputRef.size() > 1 || rhsInputRef.size() > 1) { + if (lhsInputRef.size() > 1 || rhsInputRef.size() > 1) argsRefersToBothInput = true; - } return argsRefersToBothInput; } private static Set getInputRef(Collection colDescSet, List inputRRList) { + String tableAlias; + RowResolver inputRR; Set inputLineage = new HashSet(); for (ExprNodeDesc col : colDescSet) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) col; - String tableAlias = colDesc.getTabAlias(); + tableAlias = colDesc.getTabAlias(); for (int i = 0; i < inputRRList.size(); i++) { - RowResolver inputRR = inputRRList.get(i); + inputRR = inputRRList.get(i); // If table Alias is present check if InputRR has that table and then // check for internal name // else if table alias is null then check with internal name in all // inputRR. - if (tableAlias == null) { - if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { - inputLineage.add(i); - } - } else { + if (tableAlias != null) { if (inputRR.hasTableAlias(tableAlias)) { if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { inputLineage.add(i); } } + } else { + if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { + inputLineage.add(i); + } } } } @@ -305,7 +306,7 @@ private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare /** * Factory method to get DefaultExprProcessor. - * + * * @return DefaultExprProcessor. */ @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java index df3e83f..fdee66b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java @@ -27,8 +27,7 @@ import com.google.common.collect.ImmutableList; /** - * JoinTypeCheckCtx is used by Optiq planner(CBO) to generate Join Conditions from - * Join Condition AST. + * JoinTypeCheckCtx is used by Optiq planner(CBO) to generate Join Conditions from Join Condition AST. * Reasons for sub class: * 1. Join Conditions can not handle: * a. Stateful Functions @@ -56,8 +55,8 @@ public JoinTypeCheckCtx(RowResolver leftRR, RowResolver rightRR, JoinType hiveJo throws SemanticException { super(RowResolver.getCombinedRR(leftRR, rightRR), false, false, false, false, false, false, false, false); - inputRRLst = ImmutableList.of(leftRR, rightRR); - outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) + this.inputRRLst = ImmutableList.of(leftRR, rightRR); + this.outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) || (hiveJoinType == JoinType.FULLOUTER); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java index b54b22f..006870c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java @@ -23,14 +23,15 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; -import com.google.common.collect.ImmutableMap; import net.hydromatic.avatica.ByteString; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -73,38 +74,43 @@ import org.eigenbase.sql.fun.SqlCastFunction; import org.eigenbase.sql.type.SqlTypeName; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; + public class RexNodeConverter { + private static final Log LOG = LogFactory.getLog(RexNodeConverter.class); private static class InputCtx { - private final RelDataType optiqInpDataType; - private final ImmutableMap hiveNameToPosMap; - private final RowResolver hiveRR; - private final int offsetInOptiqSchema; + private final RelDataType m_optiqInpDataType; + private final ImmutableMap m_hiveNameToPosMap; + private final RowResolver m_hiveRR; + private final int m_offsetInOptiqSchema; private InputCtx(RelDataType optiqInpDataType, ImmutableMap hiveNameToPosMap, RowResolver hiveRR, int offsetInOptiqSchema) { - this.optiqInpDataType = optiqInpDataType; - this.hiveNameToPosMap = hiveNameToPosMap; - this.hiveRR = hiveRR; - this.offsetInOptiqSchema = offsetInOptiqSchema; + m_optiqInpDataType = optiqInpDataType; + m_hiveNameToPosMap = hiveNameToPosMap; + m_hiveRR = hiveRR; + m_offsetInOptiqSchema = offsetInOptiqSchema; } - } + }; - private final RelOptCluster cluster; - private final ImmutableList inputCtxs; - private final boolean flattenExpr; + private final RelOptCluster m_cluster; + private final ImmutableList m_inputCtxs; + private final boolean m_flattenExpr; public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap nameToPosMap, int offset, boolean flattenExpr) { - this.cluster = cluster; - inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); - this.flattenExpr = flattenExpr; + this.m_cluster = cluster; + m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); + m_flattenExpr = flattenExpr; } public RexNodeConverter(RelOptCluster cluster, List inpCtxLst, boolean flattenExpr) { - this.cluster = cluster; - inputCtxs = ImmutableList.builder().addAll(inpCtxLst).build(); - this.flattenExpr = flattenExpr; + this.m_cluster = cluster; + m_inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); + m_flattenExpr = flattenExpr; } public RexNode convert(ExprNodeDesc expr) throws SemanticException { @@ -128,7 +134,7 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept RexNode rexNode = convert(fieldDesc.getDesc()); if (rexNode instanceof RexCall) { // regular case of accessing nested field in a column - return cluster.getRexBuilder().makeFieldAccess(rexNode, + return m_cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); } else { // This may happen for schema-less tables, where columns are dynamically @@ -139,50 +145,55 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept } private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticException { + ExprNodeDesc tmpExprNode; + RexNode tmpRN; TypeInfo tgtDT = null; List childRexNodeLst = new LinkedList(); - Builder argTypeBldr = ImmutableList.builder(); + Builder argTypeBldr = ImmutableList. builder(); // TODO: 1) Expand to other functions as needed 2) What about types other than primitive. if (func.getGenericUDF() instanceof GenericUDFBaseNumeric) { tgtDT = func.getTypeInfo(); - } else if (func.getGenericUDF() instanceof GenericUDFBaseCompare && - func.getChildren().size() == 2) { - tgtDT = FunctionRegistry.getCommonClassForComparison(func.getChildren().get(0).getTypeInfo(), - func.getChildren().get(1).getTypeInfo()); + } else if (func.getGenericUDF() instanceof GenericUDFBaseCompare) { + if (func.getChildren().size() == 2) { + tgtDT = FunctionRegistry.getCommonClassForComparison(func.getChildren().get(0) + .getTypeInfo(), func.getChildren().get(1).getTypeInfo()); + } } for (ExprNodeDesc childExpr : func.getChildren()) { - ExprNodeDesc tmpExprNode = childExpr; + tmpExprNode = childExpr; if (tgtDT != null && TypeInfoUtils.isConversionRequiredForComparison(tgtDT, childExpr.getTypeInfo())) { tmpExprNode = ParseUtils.createConversionCast(childExpr, (PrimitiveTypeInfo) tgtDT); } - argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory())); - RexNode tmpRN = convert(tmpExprNode); + argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), m_cluster.getTypeFactory())); + tmpRN = convert(tmpExprNode); childRexNodeLst.add(tmpRN); } // This is an explicit cast - RexNode expr = handleExplicitCast(func, childRexNodeLst); - + RexNode expr = null; RelDataType retType = null; + expr = handleExplicitCast(func, childRexNodeLst); + if (expr == null) { - retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()); + retType = (expr != null) ? expr.getType() : TypeConverter.convert(func.getTypeInfo(), + m_cluster.getTypeFactory()); SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator( func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType); - expr = cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); + expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); } else { retType = expr.getType(); } // TODO: Cast Function in Optiq have a bug where it infertype on cast throws // an exception - if (flattenExpr && (expr instanceof RexCall) + if (m_flattenExpr && (expr instanceof RexCall) && !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) { RexCall call = (RexCall) expr; - expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(), + expr = m_cluster.getRexBuilder().makeCall(retType, call.getOperator(), RexUtil.flatten(call.getOperands(), call.getOperator())); } @@ -191,19 +202,18 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep private boolean castExprUsingUDFBridge(GenericUDF gUDF) { boolean castExpr = false; - if (gUDF instanceof GenericUDFBridge) { + if (gUDF != null && gUDF instanceof GenericUDFBridge) { String udfClassName = ((GenericUDFBridge) gUDF).getUdfClassName(); if (udfClassName != null) { int sp = udfClassName.lastIndexOf('.'); // TODO: add method to UDFBridge to say if it is a cast func if (sp >= 0 & (sp + 1) < udfClassName.length()) { udfClassName = udfClassName.substring(sp + 1); - if ("UDFToBoolean".equals(udfClassName) || "UDFToByte".equals(udfClassName) - || "UDFToDouble".equals(udfClassName) || "UDFToInteger".equals(udfClassName) - || "UDFToLong".equals(udfClassName) || "UDFToShort".equals(udfClassName) - || "UDFToFloat".equals(udfClassName) || "UDFToString".equals(udfClassName)) { + if (udfClassName.equals("UDFToBoolean") || udfClassName.equals("UDFToByte") + || udfClassName.equals("UDFToDouble") || udfClassName.equals("UDFToInteger") + || udfClassName.equals("UDFToLong") || udfClassName.equals("UDFToShort") + || udfClassName.equals("UDFToFloat") || udfClassName.equals("UDFToString")) castExpr = true; - } } } } @@ -222,8 +232,8 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { // || (udf instanceof GenericUDFToUnixTimeStamp) || (udf instanceof // GenericUDFTimestamp) || castExprUsingUDFBridge(udf)) { - castExpr = cluster.getRexBuilder().makeAbstractCast( - TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()), + castExpr = m_cluster.getRexBuilder().makeAbstractCast( + TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()), childRexNodeLst.get(0)); } } @@ -234,15 +244,15 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { InputCtx ctxLookingFor = null; - if (inputCtxs.size() == 1) { - ctxLookingFor = inputCtxs.get(0); + if (m_inputCtxs.size() == 1) { + ctxLookingFor = m_inputCtxs.get(0); } else { String tableAlias = col.getTabAlias(); String colAlias = col.getColumn(); int noInp = 0; - for (InputCtx ic : inputCtxs) { - if (tableAlias == null || ic.hiveRR.hasTableAlias(tableAlias)) { - if (ic.hiveRR.getPosition(colAlias) >= 0) { + for (InputCtx ic : m_inputCtxs) { + if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) { + if (ic.m_hiveRR.getPosition(colAlias) >= 0) { ctxLookingFor = ic; noInp++; } @@ -258,16 +268,16 @@ private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException { InputCtx ic = getInputCtx(col); - int pos = ic.hiveNameToPosMap.get(col.getColumn()); - return cluster.getRexBuilder().makeInputRef( - ic.optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.offsetInOptiqSchema); + int pos = ic.m_hiveNameToPosMap.get(col.getColumn()); + return m_cluster.getRexBuilder().makeInputRef( + ic.m_optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.m_offsetInOptiqSchema); } - private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE); - private static final BigInteger MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE), + MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticException { - RexBuilder rexBuilder = cluster.getRexBuilder(); + RexBuilder rexBuilder = m_cluster.getRexBuilder(); RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory(); PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo(); RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory); @@ -278,14 +288,14 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce Object value = ObjectInspectorUtils.copyToStandardJavaObject( coi.getWritableConstantValue(), coi); - RexNode optiqLiteral; + RexNode optiqLiteral = null; // TODO: Verify if we need to use ConstantObjectInspector to unwrap data switch (hiveTypeCategory) { case BOOLEAN: - optiqLiteral = rexBuilder.makeLiteral((Boolean) value); + optiqLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue()); break; case BYTE: - byte[] byteArray = { (Byte) value}; + byte[] byteArray = new byte[] { (Byte) value}; ByteString bs = new ByteString(byteArray); optiqLiteral = rexBuilder.makeBinaryLiteral(bs); break; @@ -322,7 +332,7 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce } else { // CBO doesn't support unlimited precision decimals. In practice, this will work... // An alternative would be to throw CboSemanticException and fall back to no CBO. - RelDataType relType = cluster.getTypeFactory().createSqlType( + RelDataType relType = m_cluster.getTypeFactory().createSqlType( SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length()); optiqLiteral = rexBuilder.makeExactLiteral(bd, relType); } @@ -366,12 +376,12 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce } private RexNode createNullLiteral(ExprNodeDesc expr) throws OptiqSemanticException { - return cluster.getRexBuilder().makeNullLiteral(TypeConverter.convert( - expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName()); + return m_cluster.getRexBuilder().makeNullLiteral(TypeConverter.convert( + expr.getTypeInfo(), m_cluster.getTypeFactory()).getSqlTypeName()); } public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode, - List inputRels, Map relToHiveRR, + List inputRels, LinkedHashMap relToHiveRR, Map> relToHiveColNameOptiqPosMap, boolean flattenExpr) throws SemanticException { List inputCtxLst = new ArrayList(); @@ -383,6 +393,6 @@ public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprN offSet += r.getRowType().getFieldCount(); } - return new RexNodeConverter(cluster, inputCtxLst, flattenExpr).convert(joinCondnExprNode); + return (new RexNodeConverter(cluster, inputCtxLst, flattenExpr)).convert(joinCondnExprNode); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java index 7974b42..31f906a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java @@ -66,15 +66,15 @@ public class SqlFunctionConverter { private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); - static final Map HIVE_TO_OPTIQ; - static final Map OPTIQ_TO_HIVE_TOKENS; - static final Map REVERSE_OPERATOR_MAP; + static final Map hiveToOptiq; + static final Map optiqToHiveToken; + static final Map reverseOperatorMap; static { StaticBlockBuilder builder = new StaticBlockBuilder(); - HIVE_TO_OPTIQ = ImmutableMap.copyOf(builder.hiveToOptiq); - OPTIQ_TO_HIVE_TOKENS = ImmutableMap.copyOf(builder.optiqToHiveToken); - REVERSE_OPERATOR_MAP = ImmutableMap.copyOf(builder.reverseOperatorMap); + hiveToOptiq = ImmutableMap.copyOf(builder.hiveToOptiq); + optiqToHiveToken = ImmutableMap.copyOf(builder.optiqToHiveToken); + reverseOperatorMap = ImmutableMap.copyOf(builder.reverseOperatorMap); } public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveUDF, @@ -85,7 +85,7 @@ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveU } else if (hiveUDF instanceof GenericUDFOPPositive) { return SqlStdOperatorTable.UNARY_PLUS; } // do generic lookup - String name; + String name = null; if (StringUtils.isEmpty(funcTextName)) { name = getName(hiveUDF); // this should probably never happen, see getName comment LOG.warn("The function text was empty, name from annotation is " + name); @@ -97,14 +97,12 @@ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveU } public static GenericUDF getHiveUDF(SqlOperator op, RelDataType dt) { - String name = REVERSE_OPERATOR_MAP.get(op); - if (name == null) { + String name = reverseOperatorMap.get(op); + if (name == null) name = op.getName(); - } - FunctionInfo hFn = name == null ? null : FunctionRegistry.getFunctionInfo(name); - if (hFn == null) { + FunctionInfo hFn = name != null ? FunctionRegistry.getFunctionInfo(name) : null; + if (hFn == null) hFn = handleExplicitCast(op, dt); - } return hFn == null ? null : hFn.getGenericUDF(); } @@ -145,31 +143,31 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { FunctionRegistry.getFunctionInfo("decimal")); } else if (castType.equals(TypeInfoFactory.binaryTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("binary"); - } else { - throw new IllegalStateException("Unexpected type : " + - castType.getQualifiedName()); - } + } else throw new IllegalStateException("Unexpected type : " + + castType.getQualifiedName()); } return castUDF; } private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, FunctionInfo fi) { - SettableUDF udf = (SettableUDF) fi.getGenericUDF(); + SettableUDF udf = (SettableUDF)fi.getGenericUDF(); try { udf.setTypeInfo(ti); } catch (UDFArgumentException e) { throw new RuntimeException(e); } - return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf); + return new FunctionInfo(fi.isNative(),fi.getDisplayName(),(GenericUDF)udf); } // TODO: 1) handle Agg Func Name translation 2) is it correct to add func args // as child of func? public static ASTNode buildAST(SqlOperator op, List children) { - HiveToken hToken = OPTIQ_TO_HIVE_TOKENS.get(op); + HiveToken hToken = optiqToHiveToken.get(op); ASTNode node; - if (hToken == null) { + if (hToken != null) { + node = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); + } else { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTION, "TOK_FUNCTION"); if (op.kind != SqlKind.CAST) { if (op.kind == SqlKind.MINUS_PREFIX) { @@ -177,16 +175,15 @@ public static ASTNode buildAST(SqlOperator op, List children) { } else if (op.kind == SqlKind.PLUS_PREFIX) { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.PLUS, "PLUS"); } else { - if (op.getName().equalsIgnoreCase(SqlStdOperatorTable.COUNT.getName()) && - children.isEmpty()) { - node = - (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); + if (op.getName().toUpperCase() + .equals(SqlStdOperatorTable.COUNT.getName()) + && children.size() == 0) { + node = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); } node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName())); } } - } else { - node = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); } for (ASTNode c : children) { @@ -202,8 +199,8 @@ public static ASTNode buildAST(SqlOperator op, List children) { */ public static ASTNode buildAST(SqlOperator op, List children, int i) { if (i + 1 < children.size()) { - HiveToken hToken = OPTIQ_TO_HIVE_TOKENS.get(op); - ASTNode curNode = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); + HiveToken hToken = optiqToHiveToken.get(op); + ASTNode curNode = ((ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text)); ParseDriver.adaptor.addChild(curNode, children.get(i)); ParseDriver.adaptor.addChild(curNode, buildAST(op, children, i + 1)); return curNode; @@ -224,20 +221,19 @@ private static String getName(GenericUDF hiveUDF) { Class udfClass = hiveUDF.getClass(); Annotation udfAnnotation = udfClass.getAnnotation(Description.class); - if (udfAnnotation instanceof Description) { + if (udfAnnotation != null && udfAnnotation instanceof Description) { Description udfDescription = (Description) udfAnnotation; udfName = udfDescription.name(); if (udfName != null) { String[] aliases = udfName.split(","); - if (aliases.length > 0) { + if (aliases.length > 0) udfName = aliases[0]; - } } } if (udfName == null || udfName.isEmpty()) { udfName = hiveUDF.getClass().getName(); - int indx = udfName.lastIndexOf('.'); + int indx = udfName.lastIndexOf("."); if (indx >= 0) { indx += 1; udfName = udfName.substring(indx); @@ -291,70 +287,70 @@ private static HiveToken hToken(int type, String text) { } public static class OptiqUDAF extends SqlAggFunction { - final ImmutableList argTypes; - final RelDataType retType; + final ImmutableList m_argTypes; + final RelDataType m_retType; public OptiqUDAF(String opName, SqlReturnTypeInference returnTypeInference, SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, ImmutableList argTypes, RelDataType retType) { super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference, operandTypeInference, operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); - this.argTypes = argTypes; - this.retType = retType; + m_argTypes = argTypes; + m_retType = retType; } @Override public List getParameterTypes(final RelDataTypeFactory typeFactory) { - return argTypes; + return m_argTypes; } @Override public RelDataType getReturnType(final RelDataTypeFactory typeFactory) { - return retType; + return m_retType; } } private static class OptiqUDFInfo { - private String udfName; - private SqlReturnTypeInference returnTypeInference; - private SqlOperandTypeInference operandTypeInference; - private SqlOperandTypeChecker operandTypeChecker; - private ImmutableList argTypes; - private RelDataType retType; + private String m_udfName; + private SqlReturnTypeInference m_returnTypeInference; + private SqlOperandTypeInference m_operandTypeInference; + private SqlOperandTypeChecker m_operandTypeChecker; + private ImmutableList m_argTypes; + private RelDataType m_retType; } private static OptiqUDFInfo getUDFInfo(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { OptiqUDFInfo udfInfo = new OptiqUDFInfo(); - udfInfo.udfName = hiveUdfName; - udfInfo.returnTypeInference = ReturnTypes.explicit(optiqRetType); - udfInfo.operandTypeInference = InferTypes.explicit(optiqArgTypes); + udfInfo.m_udfName = hiveUdfName; + udfInfo.m_returnTypeInference = ReturnTypes.explicit(optiqRetType); + udfInfo.m_operandTypeInference = InferTypes.explicit(optiqArgTypes); ImmutableList.Builder typeFamilyBuilder = new ImmutableList.Builder(); for (RelDataType at : optiqArgTypes) { typeFamilyBuilder.add(Util.first(at.getSqlTypeName().getFamily(), SqlTypeFamily.ANY)); } - udfInfo.operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); + udfInfo.m_operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); - udfInfo.argTypes = ImmutableList.copyOf(optiqArgTypes); - udfInfo.retType = optiqRetType; + udfInfo.m_argTypes = ImmutableList. copyOf(optiqArgTypes); + udfInfo.m_retType = optiqRetType; return udfInfo; } - public static SqlOperator getOptiqFn(String hiveUdfName, ImmutableList optiqArgTypes, - RelDataType optiqRetType) throws OptiqSemanticException { + public static SqlOperator getOptiqFn(String hiveUdfName, + ImmutableList optiqArgTypes, RelDataType optiqRetType) throws OptiqSemanticException{ - if (hiveUdfName != null && "<=>".equals(hiveUdfName.trim())) { + if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) { // We can create Optiq IS_DISTINCT_FROM operator for this. But since our // join reordering algo cant handle this anyway there is no advantage of this. // So, bail out for now. throw new OptiqSemanticException("<=> is not yet supported for cbo."); } - SqlOperator optiqOp = HIVE_TO_OPTIQ.get(hiveUdfName); + SqlOperator optiqOp = hiveToOptiq.get(hiveUdfName); if (optiqOp == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqOp = new SqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, - uInf.operandTypeInference, uInf.operandTypeChecker, + optiqOp = new SqlFunction(uInf.m_udfName, SqlKind.OTHER_FUNCTION, uInf.m_returnTypeInference, + uInf.m_operandTypeInference, uInf.m_operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); } @@ -363,12 +359,12 @@ public static SqlOperator getOptiqFn(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { - SqlAggFunction optiqAggFn = (SqlAggFunction) HIVE_TO_OPTIQ.get(hiveUdfName); + SqlAggFunction optiqAggFn = (SqlAggFunction) hiveToOptiq.get(hiveUdfName); if (optiqAggFn == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqAggFn = new OptiqUDAF(uInf.udfName, uInf.returnTypeInference, - uInf.operandTypeInference, uInf.operandTypeChecker, uInf.argTypes, uInf.retType); + optiqAggFn = new OptiqUDAF(uInf.m_udfName, uInf.m_returnTypeInference, + uInf.m_operandTypeInference, uInf.m_operandTypeChecker, uInf.m_argTypes, uInf.m_retType); } return optiqAggFn; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java index d9d72bd..2c30e9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java @@ -51,11 +51,11 @@ import com.google.common.collect.Lists; public class TypeConverter { - private static final Map OPTIQ_TO_HIVE_TYPE_NAME_MAP; + private static final Map optiqToHiveTypeNameMap; // TODO: Handling of char[], varchar[], string... static { - Builder b = ImmutableMap.builder(); + Builder b = ImmutableMap. builder(); b.put(SqlTypeName.BOOLEAN.getName(), new HiveToken(HiveParser.TOK_BOOLEAN, "TOK_BOOLEAN")); b.put(SqlTypeName.TINYINT.getName(), new HiveToken(HiveParser.TOK_TINYINT, "TOK_TINYINT")); b.put(SqlTypeName.SMALLINT.getName(), new HiveToken(HiveParser.TOK_SMALLINT, "TOK_SMALLINT")); @@ -66,12 +66,10 @@ b.put(SqlTypeName.DATE.getName(), new HiveToken(HiveParser.TOK_DATE, "TOK_DATE")); b.put(SqlTypeName.TIMESTAMP.getName(), new HiveToken(HiveParser.TOK_TIMESTAMP, "TOK_TIMESTAMP")); b.put(SqlTypeName.BINARY.getName(), new HiveToken(HiveParser.TOK_BINARY, "TOK_BINARY")); - OPTIQ_TO_HIVE_TYPE_NAME_MAP = b.build(); - } + optiqToHiveTypeNameMap = b.build(); + }; - /** - * Convert Hive Types To Optiq Types. - */ + /*********************** Convert Hive Types To Optiq Types ***********************/ public static RelDataType getType(RelOptCluster cluster, List cInfoLst) throws OptiqSemanticException { RexBuilder rexBuilder = cluster.getRexBuilder(); @@ -104,7 +102,7 @@ public static RelDataType getType(RelOptCluster cluster, RowResolver rr, } public static RelDataType convert(TypeInfo type, RelDataTypeFactory dtFactory) - throws OptiqSemanticException { + throws OptiqSemanticException{ RelDataType convertedType = null; switch (type.getCategory()) { @@ -209,13 +207,13 @@ public static RelDataType convert(StructTypeInfo structType, final RelDataTypeFactory dtFactory) throws OptiqSemanticException { List fTypes = new ArrayList(structType.getAllStructFieldTypeInfos().size()); for (TypeInfo ti : structType.getAllStructFieldTypeInfos()) { - fTypes.add(convert(ti, dtFactory)); + fTypes.add(convert(ti,dtFactory)); } return dtFactory.createStructType(fTypes, structType.getAllStructFieldNames()); } public static RelDataType convert(UnionTypeInfo unionType, RelDataTypeFactory dtFactory) - throws OptiqSemanticException { + throws OptiqSemanticException{ // Union type is not supported in Optiq. throw new OptiqSemanticException("Union type is not supported"); } @@ -284,11 +282,10 @@ public static TypeInfo convertPrimtiveType(RelDataType rType) { case DECIMAL: return TypeInfoFactory.getDecimalTypeInfo(rType.getPrecision(), rType.getScale()); case VARCHAR: - if (rType.getPrecision() == Integer.MAX_VALUE) { + if (rType.getPrecision() == Integer.MAX_VALUE) return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - } else { + else return TypeInfoFactory.getVarcharTypeInfo(rType.getPrecision()); - } case CHAR: return TypeInfoFactory.getCharTypeInfo(rType.getPrecision()); case OTHER: @@ -298,31 +295,31 @@ public static TypeInfo convertPrimtiveType(RelDataType rType) { } - /** - * Convert Optiq Types To Hive Types. - */ + /*********************** Convert Optiq Types To Hive Types ***********************/ public static HiveToken hiveToken(RelDataType optiqType) { - HiveToken ht; + HiveToken ht = null; switch (optiqType.getSqlTypeName()) { - case CHAR: + case CHAR: { ht = new HiveToken(HiveParser.TOK_CHAR, "TOK_CHAR", String.valueOf(optiqType.getPrecision())); + } break; - case VARCHAR: - if (optiqType.getPrecision() == Integer.MAX_VALUE) { - ht = new HiveToken(HiveParser.TOK_STRING, "TOK_STRING", - String.valueOf(optiqType.getPrecision())); - } else { - ht = new HiveToken(HiveParser.TOK_VARCHAR, "TOK_VARCHAR", - String.valueOf(optiqType.getPrecision())); - } + case VARCHAR: { + if (optiqType.getPrecision() == Integer.MAX_VALUE) + ht = new HiveToken(HiveParser.TOK_STRING, "TOK_STRING", String.valueOf(optiqType + .getPrecision())); + else + ht = new HiveToken(HiveParser.TOK_VARCHAR, "TOK_VARCHAR", String.valueOf(optiqType + .getPrecision())); + } break; - case DECIMAL: + case DECIMAL: { ht = new HiveToken(HiveParser.TOK_DECIMAL, "TOK_DECIMAL", String.valueOf(optiqType .getPrecision()), String.valueOf(optiqType.getScale())); + } break; default: - ht = OPTIQ_TO_HIVE_TYPE_NAME_MAP.get(optiqType.getSqlTypeName().getName()); + ht = optiqToHiveTypeNameMap.get(optiqType.getSqlTypeName().getName()); } return ht; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index 3e2c50b..9c55379 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -352,29 +352,35 @@ public void setExpressionMap(Map expressionMap) { } - // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or not? + // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or + // not? public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, int outputColPos, int numColumns) throws SemanticException { + String tabAlias; + String colAlias; + String[] qualifiedColName; int i = 0; for (ColumnInfo cInfoFrmInput : rrToAddFrom.getRowSchema().getSignature()) { - if (numColumns >= 0 && i == numColumns) { + if ( numColumns >= 0 && i == numColumns ) { break; } - String[] qualifiedColName = rrToAddFrom.getInvRslvMap().get(cInfoFrmInput.getInternalName()); - String tabAlias = qualifiedColName[0]; - String colAlias = qualifiedColName[1]; + ColumnInfo newCI = null; + qualifiedColName = rrToAddFrom.getInvRslvMap().get( + cInfoFrmInput.getInternalName()); + tabAlias = qualifiedColName[0]; + colAlias = qualifiedColName[1]; - ColumnInfo newCI = new ColumnInfo(cInfoFrmInput); + newCI = new ColumnInfo(cInfoFrmInput); newCI.setInternalName(SemanticAnalyzer .getColumnInternalName(outputColPos)); outputColPos++; - if (rrToAddTo.get(tabAlias, colAlias) == null) { - rrToAddTo.put(tabAlias, colAlias, newCI); - } else { + if (rrToAddTo.get(tabAlias, colAlias) != null) { LOG.debug("Found duplicate column alias in RR: " + rrToAddTo.get(tabAlias, colAlias)); + } else { + rrToAddTo.put(tabAlias, colAlias, newCI); } qualifiedColName = rrToAddFrom.getAlternateMappings(cInfoFrmInput @@ -388,25 +394,30 @@ public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, } return outputColPos; - } + } public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, int outputColPos) throws SemanticException { return add(rrToAddTo, rrToAddFrom, outputColPos, -1); } - /** - * Return a new row resolver that is combination of left RR and right RR. - * The schema will be schema of left, schema of right. - */ - public static RowResolver getCombinedRR(RowResolver leftRR, - RowResolver rightRR) throws SemanticException { - int outputColPos = 0; - - RowResolver combinedRR = new RowResolver(); - outputColPos = add(combinedRR, leftRR, outputColPos); - add(combinedRR, rightRR, outputColPos); - - return combinedRR; - } + /** + * Return a new row resolver that is combination of left RR and right RR. + * The schema will be schema of left, schema of right + * + * @param leftRR + * @param rightRR + * @return + * @throws SemanticException + */ + public static RowResolver getCombinedRR(RowResolver leftRR, + RowResolver rightRR) throws SemanticException { + int outputColPos = 0; + + RowResolver combinedRR = new RowResolver(); + outputColPos = add(combinedRR, leftRR, outputColPos); + outputColPos = add(combinedRR, rightRR, outputColPos); + + return combinedRR; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2805ac8..64c468a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -249,6 +249,7 @@ import org.eigenbase.rel.rules.MergeFilterRule; import org.eigenbase.rel.rules.PushFilterPastProjectRule; import org.eigenbase.rel.rules.PushFilterPastSetOpRule; +import org.eigenbase.rel.rules.RemoveTrivialProjectRule; import org.eigenbase.rel.rules.SemiJoinRel; import org.eigenbase.rel.rules.TransitivePredicatesOnJoinRule; import org.eigenbase.relopt.RelOptCluster; @@ -283,6 +284,7 @@ import org.eigenbase.util.CompositeList; import org.eigenbase.util.ImmutableIntList; import org.eigenbase.util.Pair; +import org.eigenbase.util.Util; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -1201,9 +1203,8 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) posn++; } - if (ast.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM) { + if ((ast.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM)) queryProperties.setUsesScript(true); - } LinkedHashMap aggregations = doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest); @@ -1215,9 +1216,8 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) case HiveParser.TOK_WHERE: qbp.setWhrExprForClause(ctx_1.dest, ast); - if (!SubQueryUtils.findSubQueries((ASTNode) ast.getChild(0)).isEmpty()) { - queryProperties.setFilterWithSubQuery(true); - } + if (!SubQueryUtils.findSubQueries((ASTNode) ast.getChild(0)).isEmpty()) + queryProperties.setFilterWithSubQuery(true); break; case HiveParser.TOK_INSERT_INTO: @@ -2430,7 +2430,7 @@ private Operator genHavingPlan(String dest, QB qb, Operator input, output = putOpInsertMap(output, inputRR); return output; } - + private Operator genPlanForSubQueryPredicate( QB qbSQ, ISubQueryJoinInfo subQueryPredicate) throws SemanticException { @@ -9835,11 +9835,9 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (runCBO) { boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY || ast.getToken().getType() == HiveParser.TOK_EXPLAIN; - if (!tokenTypeIsQuery || - createVwDesc != null || - !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) || - !canHandleQuery(qb, true) || - !HiveOptiqUtil.validateASTForCBO(ast)) { + if (!tokenTypeIsQuery || createVwDesc != null + || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) + || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) { runCBO = false; } @@ -9910,12 +9908,11 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { sinkOp = genPlan(qb); } - if (createVwDesc != null) { + if (createVwDesc != null) resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver()); - } else { + else resultSchema = convertRowSchemaToResultSetSchema(opParseCtx.get(sinkOp).getRowResolver(), - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); - } + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); ParseContext pCtx = new ParseContext(conf, qb, child, opToPartPruner, opToPartList, topOps, topSelOps, opParseCtx, joinContext, smbMapJoinContext, @@ -11291,10 +11288,10 @@ private OrderSpec processOrderSpec(ASTNode sortNode) { OrderExpression exprSpec = new OrderExpression(); exprSpec.setExpression((ASTNode) sortNode.getChild(i).getChild(0)); if ( sortNode.getChild(i).getType() == HiveParser.TOK_TABSORTCOLNAMEASC ) { - exprSpec.setOrder(PTFInvocationSpec.Order.ASC); + exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.ASC); } else { - exprSpec.setOrder(PTFInvocationSpec.Order.DESC); + exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.DESC); } oSpec.addExpression(exprSpec); } @@ -11663,7 +11660,7 @@ void buildPTFReduceSinkDetails(PartitionedTableFunctionDef tabDef, List orderColList = tabDef.getOrder().getExpressions(); for (int i = 0; i < orderColList.size(); i++) { OrderExpressionDef colDef = orderColList.get(i); - PTFInvocationSpec.Order order = colDef.getOrder(); + org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order order = colDef.getOrder(); if (order.name().equals("ASC")) { orderString.append('+'); } else { @@ -11911,7 +11908,7 @@ private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, spec.getQueryOrderSpec().getExpressions(); for (int i = 0; i < orderColList.size(); i++) { OrderExpression orderCol = orderColList.get(i); - PTFInvocationSpec.Order order = orderCol.getOrder(); + org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order order = orderCol.getOrder(); if (order.name().equals("ASC")) { orderString.append('+'); } else { @@ -12149,7 +12146,7 @@ protected boolean deleting() { return false; } - /* Temporary Place Holder For Optiq plan Gen, Optimizer */ + /**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/ /* * Entry point to Optimizations using Optiq. @@ -12159,19 +12156,13 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { // Assumption: // 1. If top level QB is query then everything below it must also be Query // 2. Nested Subquery will return false for qbToChk.getIsQuery() - if ((!topLevelQB || qbToChk.getIsQuery()) && - (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || - conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict")) && - (!topLevelQB || - (queryProperties.getJoinCount() > 1) || - conf.getBoolVar(ConfVars.HIVE_IN_TEST)) && - !queryProperties.hasClusterBy() && - !queryProperties.hasDistributeBy() && - !queryProperties.hasSortBy() && - !queryProperties.hasPTF() && - !queryProperties.usesScript() && - !queryProperties.hasMultiDestQuery() && - !queryProperties.hasLateralViews()) { + if ((!topLevelQB || qbToChk.getIsQuery()) + && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict")) + && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST)) + && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy() + && !queryProperties.hasSortBy() && !queryProperties.hasPTF() + && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery() + && !queryProperties.hasLateralViews()) { runOptiqPlanner = true; } else { LOG.info("Can not invoke CBO; query contains operators not supported for CBO."); @@ -12181,25 +12172,24 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { } private class OptiqBasedPlanner implements Frameworks.PlannerAction { - - RelOptCluster cluster; - RelOptSchema relOptSchema; - SemanticException semanticException; - Map partitionCache; - AtomicInteger noColsMissingStats = new AtomicInteger(0); + RelOptCluster cluster; + RelOptSchema relOptSchema; + SemanticException semanticException; + Map partitionCache; + AtomicInteger noColsMissingStats = new AtomicInteger(0); List topLevelFieldSchema; // TODO: Do we need to keep track of RR, ColNameToPosMap for every op or // just last one. - Map relToHiveRR = new LinkedHashMap(); - Map> relToHiveColNameOptiqPosMap = - new LinkedHashMap>(); + LinkedHashMap relToHiveRR = new LinkedHashMap(); + LinkedHashMap> relToHiveColNameOptiqPosMap = new LinkedHashMap>(); private ASTNode getOptimizedAST(Map partitionCache) throws SemanticException { + ASTNode optiqOptimizedAST = null; + RelNode optimizedOptiqPlan = null; this.partitionCache = partitionCache; - RelNode optimizedOptiqPlan = null; try { optimizedOptiqPlan = Frameworks.withPlanner(this, Frameworks.newConfigBuilder().typeSystem(new HiveTypeSystemImpl()).build()); @@ -12209,13 +12199,16 @@ private ASTNode getOptimizedAST(Map partitionCache) else throw new RuntimeException(e); } - ASTNode optiqOptimizedAST = ASTConverter.convert(optimizedOptiqPlan, topLevelFieldSchema); + optiqOptimizedAST = ASTConverter.convert(optimizedOptiqPlan, topLevelFieldSchema); return optiqOptimizedAST; } @Override public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) { + RelNode optiqGenPlan = null; + RelNode optiqPreCboPlan = null; + RelNode optiqOptimizedPlan = null; /* * recreate cluster, so that it picks up the additional traitDef @@ -12228,7 +12221,6 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu this.cluster = cluster; this.relOptSchema = relOptSchema; - RelNode optiqGenPlan = null; try { optiqGenPlan = genLogicalPlan(qb); topLevelFieldSchema = convertRowSchemaToResultSetSchema(relToHiveRR.get(optiqGenPlan), @@ -12238,18 +12230,18 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu throw new RuntimeException(e); } - RelNode optiqPreCboPlan = - applyPreCBOTransforms(optiqGenPlan, HiveDefaultRelMetadataProvider.INSTANCE); + optiqPreCboPlan = applyPreCBOTransforms(optiqGenPlan, HiveDefaultRelMetadataProvider.INSTANCE); List list = Lists.newArrayList(); list.add(HiveDefaultRelMetadataProvider.INSTANCE); RelTraitSet desiredTraits = cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); + HepProgram hepPgm = null; HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP) .addRuleInstance(new ConvertMultiJoinRule(HiveJoinRel.class)); hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoinRel.HIVE_JOIN_FACTORY, HiveProjectRel.DEFAULT_PROJECT_FACTORY, HiveFilterRel.DEFAULT_FILTER_FACTORY)); - HepProgram hepPgm = hepPgmBldr.build(); + hepPgm = hepPgmBldr.build(); HepPlanner hepPlanner = new HepPlanner(hepPgm); hepPlanner.registerMetadataProviders(list); @@ -12263,7 +12255,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu } hepPlanner.setRoot(rootRel); - RelNode optiqOptimizedPlan = hepPlanner.findBestExp(); + optiqOptimizedPlan = hepPlanner.findBestExp(); if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { LOG.debug("CBO Planning details:\n"); @@ -12294,13 +12286,11 @@ public RelNode applyPreCBOTransforms(RelNode basePlan, RelMetadataProvider mdPro JoinRelBase.class, HiveFilterRel.DEFAULT_FILTER_FACTORY), // TODO: Enable it after OPTIQ-407 is fixed //RemoveTrivialProjectRule.INSTANCE, - new HivePartitionPrunerRule(conf)); + new HivePartitionPrunerRule(SemanticAnalyzer.this.conf)); - RelFieldTrimmer fieldTrimmer = - new RelFieldTrimmer(null, HiveProjectRel.DEFAULT_PROJECT_FACTORY, - HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveJoinRel.HIVE_JOIN_FACTORY, - RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSortRel.HIVE_SORT_REL_FACTORY, - HiveAggregateRel.HIVE_AGGR_REL_FACTORY, HiveUnionRel.UNION_REL_FACTORY); + RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProjectRel.DEFAULT_PROJECT_FACTORY, + HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveJoinRel.HIVE_JOIN_FACTORY, RelFactories.DEFAULT_SEMI_JOIN_FACTORY, + HiveSortRel.HIVE_SORT_REL_FACTORY, HiveAggregateRel.HIVE_AGGR_REL_FACTORY, HiveUnionRel.UNION_REL_FACTORY); basePlan = fieldTrimmer.trim(basePlan); return basePlan; @@ -12309,7 +12299,7 @@ public RelNode applyPreCBOTransforms(RelNode basePlan, RelMetadataProvider mdPro private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, RelOptRule... rules) { - + RelNode optimizedRelNode = basePlan; HepProgramBuilder programBuilder = new HepProgramBuilder(); if (followPlanChanges) { programBuilder.addMatchOrder(HepMatchOrder.TOP_DOWN); @@ -12329,19 +12319,22 @@ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, new CachingRelMetadataProvider(chainedProvider, planner)); planner.setRoot(basePlan); - return planner.findBestExp(); + optimizedRelNode = planner.findBestExp(); + + return optimizedRelNode; } @SuppressWarnings("nls") private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode leftRel, String rightalias, RelNode rightRel) throws SemanticException { + HiveUnionRel unionRel = null; // 1. Get Row Resolvers, Column map for original left and right input of // Union Rel - RowResolver leftRR = relToHiveRR.get(leftRel); - RowResolver rightRR = relToHiveRR.get(rightRel); - Map leftmap = leftRR.getFieldMap(leftalias); - Map rightmap = rightRR.getFieldMap(rightalias); + RowResolver leftRR = this.relToHiveRR.get(leftRel); + RowResolver rightRR = this.relToHiveRR.get(rightRel); + HashMap leftmap = leftRR.getFieldMap(leftalias); + HashMap rightmap = rightRR.getFieldMap(rightalias); // 2. Validate that Union is feasible according to Hive (by using type // info from RR) @@ -12407,29 +12400,31 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode List leftRowDT = leftRel.getRowType().getFieldList(); List rightRowDT = rightRel.getRowType().getFieldList(); + RelDataType leftFieldDT; + RelDataType rightFieldDT; + RelDataType unionFieldDT; for (int i = 0; i < leftRowDT.size(); i++) { - RelDataType leftFieldDT = leftRowDT.get(i).getType(); - RelDataType rightFieldDT = rightRowDT.get(i).getType(); - if (leftFieldDT.equals(rightFieldDT)) { - leftProjs.add(cluster.getRexBuilder() - .ensureType(leftFieldDT, cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); - rightProjs.add(cluster.getRexBuilder() - .ensureType(rightFieldDT, cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); - } else { - RelDataType unionFieldDT = - TypeConverter.convert(unionoutRR.getColumnInfos().get(i).getType(), - cluster.getTypeFactory()); + leftFieldDT = leftRowDT.get(i).getType(); + rightFieldDT = rightRowDT.get(i).getType(); + if (!leftFieldDT.equals(rightFieldDT)) { + unionFieldDT = TypeConverter.convert(unionoutRR.getColumnInfos().get(i).getType(), + cluster.getTypeFactory()); if (!unionFieldDT.equals(leftFieldDT)) { leftNeedsTypeCast = true; } - leftProjs.add(cluster.getRexBuilder() - .ensureType(unionFieldDT, cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); + leftProjs.add(cluster.getRexBuilder().ensureType(unionFieldDT, + cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); if (!unionFieldDT.equals(rightFieldDT)) { rightNeedsTypeCast = true; } - rightProjs.add(cluster.getRexBuilder() - .ensureType(unionFieldDT, cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); + rightProjs.add(cluster.getRexBuilder().ensureType(unionFieldDT, + cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); + } else { + leftProjs.add(cluster.getRexBuilder().ensureType(leftFieldDT, + cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); + rightProjs.add(cluster.getRexBuilder().ensureType(rightFieldDT, + cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); } } @@ -12447,32 +12442,34 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode } // 6. Construct Union Rel - ImmutableList.Builder bldr = new ImmutableList.Builder(); + ImmutableList.Builder bldr = new ImmutableList.Builder(); bldr.add(unionLeftInput); bldr.add(unionRightInput); - HiveUnionRel unionRel = - new HiveUnionRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), bldr.build()); + unionRel = new HiveUnionRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), + bldr.build()); relToHiveRR.put(unionRel, unionoutRR); - relToHiveColNameOptiqPosMap.put(unionRel, buildHiveToOptiqColumnMap(unionoutRR, unionRel)); + relToHiveColNameOptiqPosMap.put(unionRel, + this.buildHiveToOptiqColumnMap(unionoutRR, unionRel)); return unionRel; } private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJoinType, ASTNode joinCond) throws SemanticException { + RelNode joinRel = null; // 1. construct the RowResolver for the new Join Node by combining row // resolvers from left, right - RowResolver leftRR = relToHiveRR.get(leftRel); - RowResolver rightRR = relToHiveRR.get(rightRel); - RowResolver joinRR; + RowResolver leftRR = this.relToHiveRR.get(leftRel); + RowResolver rightRR = this.relToHiveRR.get(rightRel); + RowResolver joinRR = null; - if (hiveJoinType == JoinType.LEFTSEMI) { + if (hiveJoinType != JoinType.LEFTSEMI) { + joinRR = RowResolver.getCombinedRR(leftRR, rightRR); + } else { joinRR = new RowResolver(); RowResolver.add(joinRR, leftRR, 0); - } else { - joinRR = RowResolver.getCombinedRR(leftRR, rightRR); } // 2. Construct ExpressionNodeDesc representing Join Condition @@ -12481,10 +12478,9 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, jCtx); - if (jCtx.getError() != null) { - throw new SemanticException( - generateErrorMessage(jCtx.getErrorSrcNode(), jCtx.getError())); - } + if (jCtx.getError() != null) + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(jCtx.getErrorSrcNode(), + jCtx.getError())); ExprNodeDesc joinCondnExprNode = exprNodes.get(joinCond); @@ -12525,7 +12521,6 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ break; } - RelNode joinRel = null; if (leftSemiJoin) { List sysFieldList = new ArrayList(); List leftJoinKeys = new ArrayList(); @@ -12539,7 +12534,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ + nonEquiConds); } - RelNode[] inputRels = { leftRel, rightRel }; + RelNode[] inputRels = new RelNode[] { leftRel, rightRel }; final List leftKeys = new ArrayList(); final List rightKeys = new ArrayList(); optiqJoinCond = HiveOptiqUtil.projectNonColumnEquiConditions( @@ -12563,23 +12558,28 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ /** * Generate Join Logical Plan Relnode by walking through the join AST. * + * @param qb * @param aliasToRel * Alias(Table/Relation alias) to RelNode; only read and not * written in to by this method + * @return + * @throws SemanticException */ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map aliasToRel) throws SemanticException { + RelNode leftRel = null; + RelNode rightRel = null; + JoinType hiveJoinType = null; if (joinParseTree.getToken().getType() == HiveParser.TOK_UNIQUEJOIN) { - String msg = "UNIQUE JOIN is currently not supported in CBO," - + " turn off cbo to use UNIQUE JOIN."; + String msg = String.format("UNIQUE JOIN is currently not supported in CBO," + + " turn off cbo to use UNIQUE JOIN."); LOG.debug(msg); throw new OptiqSemanticException(msg); } // 1. Determine Join Type // TODO: What about TOK_CROSSJOIN, TOK_MAPJOIN - JoinType hiveJoinType = null; switch (joinParseTree.getToken().getType()) { case HiveParser.TOK_LEFTOUTERJOIN: hiveJoinType = JoinType.LEFTOUTER; @@ -12600,7 +12600,6 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // 2. Get Left Table Alias ASTNode left = (ASTNode) joinParseTree.getChild(0); - RelNode leftRel = null; if ((left.getToken().getType() == HiveParser.TOK_TABREF) || (left.getToken().getType() == HiveParser.TOK_SUBQUERY) || (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { @@ -12611,19 +12610,18 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) // guranteed to have an lias here: check done in processJoin - leftTableAlias = - (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier( - left.getChild(1).getText().toLowerCase()) : leftTableAlias; + leftTableAlias = (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier(left + .getChild(1).getText().toLowerCase()) + : leftTableAlias; leftRel = aliasToRel.get(leftTableAlias); } else if (isJoinToken(left)) { leftRel = genJoinLogicalPlan(left, aliasToRel); } else { - assert false; + assert (false); } // 3. Get Right Table Alias ASTNode right = (ASTNode) joinParseTree.getChild(1); - RelNode rightRel = null; if ((right.getToken().getType() == HiveParser.TOK_TABREF) || (right.getToken().getType() == HiveParser.TOK_SUBQUERY) || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { @@ -12634,12 +12632,12 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) // guranteed to have an lias here: check done in processJoin - rightTableAlias = - (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier( - right.getChild(1).getText().toLowerCase()) : rightTableAlias; + rightTableAlias = (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier(right + .getChild(1).getText().toLowerCase()) + : rightTableAlias; rightRel = aliasToRel.get(rightTableAlias); } else { - assert false; + assert (false); } // 4. Get Join Condn @@ -12659,14 +12657,14 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc if ( qb.getParseInfo().getTabSample(tableAlias) != null || SemanticAnalyzer.this.nameToSplitSample.containsKey(tableAlias)) { String msg = String.format("Table Sample specified for %s." + - " Currently we don't support Table Sample clauses in CBO," + - " turn off cbo for queries on tableSamples.", tableAlias); + " Currently we don't support Table Sample clauses in CBO," + + " turn off cbo for queries on tableSamples.", tableAlias); LOG.debug(msg); throw new OptiqSemanticException(msg); } // 1. Get Table Alias - String aliasId = getAliasId(tableAlias, qb); + String alias_id = getAliasId(tableAlias, qb); // 2. Get Table Metadata Table tab = qb.getMetaData().getSrcForAlias(tableAlias); @@ -12682,12 +12680,13 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ColumnInfo colInfo; String colName; ArrayList cInfoLst = new ArrayList(); - for (StructField field : fields) { - colName = field.getFieldName(); - colInfo = new ColumnInfo(field.getFieldName(), - TypeInfoUtils.getTypeInfoFromObjectInspector(field.getFieldObjectInspector()), - tableAlias, false); - colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); + for (int i = 0; i < fields.size(); i++) { + colName = fields.get(i).getFieldName(); + colInfo = new ColumnInfo( + fields.get(i).getFieldName(), + TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()), + tableAlias, false); + colInfo.setSkewedCol((isSkewedCol(tableAlias, qb, colName)) ? true : false); rr.put(tableAlias, colName, colInfo); cInfoLst.add(colInfo); } @@ -12696,19 +12695,21 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema partCol : tab.getPartCols()) { - colName = partCol.getName(); + for (FieldSchema part_col : tab.getPartCols()) { + colName = part_col.getName(); colInfo = new ColumnInfo(colName, - TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), tableAlias, true); + TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), tableAlias, true); rr.put(tableAlias, colName, colInfo); cInfoLst.add(colInfo); partitionColumns.add(colInfo); } // 3.3 Add column info corresponding to virtual columns - for (VirtualColumn vc : VirtualColumn.getRegistry(conf)) { - colInfo = - new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true, vc.getIsHidden()); + Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); + while (vcs.hasNext()) { + VirtualColumn vc = vcs.next(); + colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true, + vc.getIsHidden()); rr.put(tableAlias, vc.getName(), colInfo); cInfoLst.add(colInfo); } @@ -12729,10 +12730,10 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc relToHiveRR.put(tableRel, rr); relToHiveColNameOptiqPosMap.put(tableRel, hiveToOptiqColMap); } catch (Exception e) { - if (e instanceof SemanticException) { + if ( e instanceof SemanticException) { throw (SemanticException) e; } else { - throw new RuntimeException(e); + throw (new RuntimeException(e)); } } @@ -12770,7 +12771,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, */ int numSrcColumns = srcRel.getRowType().getFieldCount(); List subQueriesInOriginalTree = SubQueryUtils.findSubQueries(searchCond); - if (!subQueriesInOriginalTree.isEmpty()) { + if (subQueriesInOriginalTree.size() > 0) { /* * Restriction.9.m :: disallow nested SubQuery expressions. @@ -12810,10 +12811,10 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QBSubQuery subQuery = SubQueryUtils.buildSubQuery(qb.getId(), sqIdx, subQueryAST, originalSubQueryAST, ctx); - if (forHavingClause) { - qb.setHavingClauseSubQueryPredicate(subQuery); - } else { + if (!forHavingClause) { qb.setWhereClauseSubQueryPredicate(subQuery); + } else { + qb.setHavingClauseSubQueryPredicate(subQuery); } String havingInputAlias = null; @@ -12827,8 +12828,8 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true); qbSQ.setSubQueryDef(subQuery.getSubQuery()); - Phase1Ctx ctx1 = initPhase1Ctx(); - doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx1); + Phase1Ctx ctx_1 = initPhase1Ctx(); + doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1); getMetaData(qbSQ); RelNode subQueryRelNode = genLogicalPlan(qbSQ); aliasToRel.put(subQuery.getAlias(), subQueryRelNode); @@ -12853,12 +12854,12 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, if (subQuery.getNotInCheck() != null) { QBSubQuery.NotInCheck notInCheck = subQuery.getNotInCheck(); notInCheck.setSQRR(sqRR); - QB qbSqNic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); - qbSqNic.setSubQueryDef(notInCheck.getSubQuery()); - ctx1 = initPhase1Ctx(); - doPhase1(notInCheck.getSubQueryAST(), qbSqNic, ctx1); - getMetaData(qbSqNic); - RelNode subQueryNICRelNode = genLogicalPlan(qbSqNic); + QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); + qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery()); + ctx_1 = initPhase1Ctx(); + doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1); + getMetaData(qbSQ_nic); + RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic); aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode); srcRel = genJoinRelNode(srcRel, subQueryNICRelNode, // set explicitly to inner until we figure out SemiJoin use @@ -12938,17 +12939,17 @@ private RelNode genFilterLogicalPlan(QB qb, RelNode srcRel, Map * Class to store GenericUDAF related information. */ private class AggInfo { - private final List aggParams; - private final TypeInfo returnType; - private final String udfName; - private final boolean distinct; + private final List m_aggParams; + private final TypeInfo m_returnType; + private final String m_udfName; + private final boolean m_distinct; private AggInfo(List aggParams, TypeInfo returnType, String udfName, boolean isDistinct) { - this.aggParams = aggParams; - this.returnType = returnType; - this.udfName = udfName; - distinct = isDistinct; + m_aggParams = aggParams; + m_returnType = returnType; + m_udfName = udfName; + m_distinct = isDistinct; } } @@ -12957,19 +12958,20 @@ private AggregateCall convertGBAgg(AggInfo agg, RelNode input, List gbC Integer childProjLstIndx) throws SemanticException { // 1. Get agg fn ret type in Optiq - RelDataType aggFnRetType = TypeConverter.convert(agg.returnType, + RelDataType aggFnRetType = TypeConverter.convert(agg.m_returnType, this.cluster.getTypeFactory()); // 2. Convert Agg Fn args and type of args to Optiq // TODO: Does HQL allows expressions as aggregate args or can it only be // projections from child? + Integer inputIndx; List argList = new ArrayList(); - RexNode rexNd; - RelDataTypeFactory dtFactory = cluster.getTypeFactory(); + RexNode rexNd = null; + RelDataTypeFactory dtFactory = this.cluster.getTypeFactory(); ImmutableList.Builder aggArgRelDTBldr = new ImmutableList.Builder(); - for (ExprNodeDesc expr : agg.aggParams) { + for (ExprNodeDesc expr : agg.m_aggParams) { rexNd = converter.convert(expr); - Integer inputIndx = rexNodeToPosMap.get(rexNd.toString()); + inputIndx = rexNodeToPosMap.get(rexNd.toString()); if (inputIndx == null) { gbChildProjLst.add(rexNd); rexNodeToPosMap.put(rexNd.toString(), childProjLstIndx); @@ -12984,26 +12986,27 @@ private AggregateCall convertGBAgg(AggInfo agg, RelNode input, List gbC // 3. Get Aggregation FN from Optiq given name, ret type and input arg // type - final Aggregation aggregation = SqlFunctionConverter.getOptiqAggFn(agg.udfName, + final Aggregation aggregation = SqlFunctionConverter.getOptiqAggFn(agg.m_udfName, aggArgRelDTBldr.build(), aggFnRetType); - return new AggregateCall(aggregation, agg.distinct, argList, aggFnRetType, null); + return new AggregateCall(aggregation, agg.m_distinct, argList, aggFnRetType, null); } private RelNode genGBRelNode(List gbExprs, List aggInfoLst, RelNode srcRel) throws SemanticException { - RowResolver gbInputRR = relToHiveRR.get(srcRel); + RowResolver gbInputRR = this.relToHiveRR.get(srcRel); ArrayList signature = gbInputRR.getRowSchema().getSignature(); - ImmutableMap posMap = relToHiveColNameOptiqPosMap.get(srcRel); - RexNodeConverter converter = new RexNodeConverter(cluster, srcRel.getRowType(), + ImmutableMap posMap = this.relToHiveColNameOptiqPosMap.get(srcRel); + RexNodeConverter converter = new RexNodeConverter(this.cluster, srcRel.getRowType(), posMap, 0, false); final List gbChildProjLst = Lists.newArrayList(); final HashMap rexNodeToPosMap = new HashMap(); final BitSet groupSet = new BitSet(); Integer gbIndx = 0; + RexNode rnd; for (ExprNodeDesc key : gbExprs) { - RexNode rnd = converter.convert(key); + rnd = converter.convert(key); gbChildProjLst.add(rnd); groupSet.set(gbIndx); rexNodeToPosMap.put(rnd.toString(), gbIndx); @@ -13011,6 +13014,7 @@ private RelNode genGBRelNode(List gbExprs, List aggInfoLs } List aggregateCalls = Lists.newArrayList(); + int i = aggInfoLst.size(); for (AggInfo agg : aggInfoLst) { aggregateCalls.add(convertGBAgg(agg, srcRel, gbChildProjLst, converter, rexNodeToPosMap, gbChildProjLst.size())); @@ -13038,13 +13042,13 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, RowResolver gByInputRR, RowResolver gByRR) { if (gByExpr.getType() == HiveParser.DOT && gByExpr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL) { - String tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getChild(0) + String tab_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getChild(0) .getText()); - String colAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(1).getText()); - gByRR.put(tabAlias, colAlias, colInfo); + String col_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(1).getText()); + gByRR.put(tab_alias, col_alias, colInfo); } else if (gByExpr.getType() == HiveParser.TOK_TABLE_OR_COL) { - String colAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getText()); - String tabAlias = null; + String col_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getText()); + String tab_alias = null; /* * If the input to the GBy has a tab alias for the column, then add an * entry based on that tab_alias. For e.g. this query: select b.x, @@ -13053,11 +13057,11 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, * ancestor before any GBy/ReduceSinks added for the GBY operation. */ try { - ColumnInfo pColInfo = gByInputRR.get(tabAlias, colAlias); - tabAlias = pColInfo == null ? null : pColInfo.getTabAlias(); + ColumnInfo pColInfo = gByInputRR.get(tab_alias, col_alias); + tab_alias = pColInfo == null ? null : pColInfo.getTabAlias(); } catch (SemanticException se) { } - gByRR.put(tabAlias, colAlias, colInfo); + gByRR.put(tab_alias, col_alias, colInfo); } } @@ -13080,6 +13084,7 @@ private void addToGBExpr(RowResolver groupByOutputRowResolver, private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver inputRR) throws SemanticException { + AggInfo aInfo = null; // 1 Convert UDAF Params to ExprNodeDesc ArrayList aggParameters = new ArrayList(); @@ -13100,7 +13105,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver // 3.2 Rank functions type is 'int'/'double' if (FunctionRegistry.isRankingFunction(aggName)) { - if ("percent_rank".equalsIgnoreCase(aggName)) + if (aggName.equalsIgnoreCase("percent_rank")) udafRetType = TypeInfoFactory.doubleTypeInfo; else udafRetType = TypeInfoFactory.intTypeInfo; @@ -13115,7 +13120,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver GenericUDAFEvaluator genericUDAFEvaluator = null; if (aggName.toLowerCase().equals(FunctionRegistry.LEAD_FUNC_NAME) || aggName.toLowerCase().equals(FunctionRegistry.LAG_FUNC_NAME)) { - List originalParameterTypeInfos = + ArrayList originalParameterTypeInfos = getWritableObjectInspector(aggParameters); genericUDAFEvaluator = FunctionRegistry.getGenericWindowingEvaluator(aggName, @@ -13125,7 +13130,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver } else { genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, aggParameters, aggAst, isDistinct, isAllColumns); - assert genericUDAFEvaluator != null; + assert (genericUDAFEvaluator != null); // 3.3.2 Get UDAF Info using UDAF Evaluator GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); @@ -13148,7 +13153,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver } // 4. Construct AggInfo - AggInfo aInfo = new AggInfo(aggParameters, udafRetType, aggName, isDistinct); + aInfo = new AggInfo(aggParameters, udafRetType, aggName, isDistinct); return aInfo; } @@ -13156,9 +13161,13 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver /** * Generate GB plan. * - * TODO: 1. Grouping Sets (roll up..) + * @param qb + * @param srcRel + * @return TODO: 1. Grouping Sets (roll up..) + * @throws SemanticException */ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { + RelNode gbRel = null; QBParseInfo qbp = getQBParseInfo(qb); // 0. for GSets, Cube, Rollup, bail from Optiq path. @@ -13166,10 +13175,10 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty()) { String gbyClause = null; - Map gbysMap = qbp.getDestToGroupBy(); + HashMap gbysMap = qbp.getDestToGroupBy(); if (gbysMap.size() == 1) { ASTNode gbyAST = gbysMap.entrySet().iterator().next().getValue(); - gbyClause = ctx.getTokenRewriteStream() + gbyClause = SemanticAnalyzer.this.ctx.getTokenRewriteStream() .toString(gbyAST.getTokenStartIndex(), gbyAST.getTokenStopIndex()); gbyClause = "in '" + gbyClause + "'."; @@ -13188,14 +13197,14 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // NOTE: Multi Insert is not supported String detsClauseName = qbp.getClauseNames().iterator().next(); List grpByAstExprs = getGroupByForClause(qbp, detsClauseName); - Map aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); - boolean hasGrpByAstExprs = grpByAstExprs != null && !grpByAstExprs.isEmpty(); - boolean hasAggregationTrees = aggregationTrees != null && !aggregationTrees.isEmpty(); + HashMap aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); + boolean hasGrpByAstExprs = (grpByAstExprs != null && !grpByAstExprs.isEmpty()) ? true : false; + boolean hasAggregationTrees = (aggregationTrees != null && !aggregationTrees.isEmpty()) ? true + : false; - RelNode gbRel = null; if (hasGrpByAstExprs || hasAggregationTrees) { - List gbExprNDescLst = new ArrayList(); - List outputColumnNames = new ArrayList(); + ArrayList gbExprNDescLst = new ArrayList(); + ArrayList outputColumnNames = new ArrayList(); int numDistinctUDFs = 0; // 2. Input, Output Row Resolvers @@ -13205,24 +13214,23 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException if (hasGrpByAstExprs) { // 3. Construct GB Keys (ExprNode) - for (ASTNode grpbyExpr : grpByAstExprs) { - Map astToExprNDescMap = - TypeCheckProcFactory.genExprNode(grpbyExpr, - new TypeCheckCtx(groupByInputRowResolver)); + for (int i = 0; i < grpByAstExprs.size(); ++i) { + ASTNode grpbyExpr = grpByAstExprs.get(i); + Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( + grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); - if (grpbyExprNDesc == null) { + if (grpbyExprNDesc == null) throw new RuntimeException("Invalid Column Reference: " + grpbyExpr.dump()); - } addToGBExpr(groupByOutputRowResolver, groupByInputRowResolver, grpbyExpr, - grpbyExprNDesc, gbExprNDescLst, outputColumnNames); + grpbyExprNDesc, gbExprNDescLst, outputColumnNames); } } // 4. Construct aggregation function Info - List aggregations = new ArrayList(); + ArrayList aggregations = new ArrayList(); if (hasAggregationTrees) { - assert aggregationTrees != null; + assert (aggregationTrees != null); for (ASTNode value : aggregationTrees.values()) { // 4.1 Determine type of UDAF // This is the GenericUDAF name @@ -13244,13 +13252,13 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, aggParameters, value, isDistinct, isAllColumns); - assert genericUDAFEvaluator != null; + assert (genericUDAFEvaluator != null); GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); AggInfo aInfo = new AggInfo(aggParameters, udaf.returnType, aggName, isDistinct); aggregations.add(aInfo); String field = getColumnInternalName(gbExprNDescLst.size() + aggregations.size() - 1); outputColumnNames.add(field); - groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.returnType, + groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.m_returnType, "", false)); } } @@ -13275,7 +13283,7 @@ private RelNode genOBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 1. OB Expr sanity test // in strict mode, in the presence of order by, limit must be specified Integer limit = qb.getParseInfo().getDestLimit(dest); - if ("strict".equalsIgnoreCase(conf.getVar(ConfVars.HIVEMAPREDMODE)) + if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") && limit == null) { throw new SemanticException(generateErrorMessage(obAST, ErrorMsg.NO_LIMIT_WITH_ORDERBY.getMsg())); @@ -13288,26 +13296,27 @@ private RelNode genOBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException int fieldIndex = 0; List obASTExprLst = obAST.getChildren(); + ASTNode obASTExpr; List> vcASTTypePairs = new ArrayList>(); RowResolver inputRR = relToHiveRR.get(srcRel); RowResolver outputRR = new RowResolver(); + RexNode rnd; RexNodeConverter converter = new RexNodeConverter(cluster, srcRel.getRowType(), relToHiveColNameOptiqPosMap.get(srcRel), 0, false); int srcRelRecordSz = srcRel.getRowType().getFieldCount(); - for (Node anObASTExprLst : obASTExprLst) { + for (int i = 0; i < obASTExprLst.size(); i++) { // 2.1 Convert AST Expr to ExprNode - ASTNode obASTExpr = (ASTNode) anObASTExprLst; - Map astToExprNDescMap = - TypeCheckProcFactory.genExprNode(obASTExpr, new TypeCheckCtx(inputRR)); + obASTExpr = (ASTNode) obASTExprLst.get(i); + Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( + obASTExpr, new TypeCheckCtx(inputRR)); ExprNodeDesc obExprNDesc = astToExprNDescMap.get(obASTExpr.getChild(0)); - if (obExprNDesc == null) { + if (obExprNDesc == null) throw new SemanticException("Invalid order by expression: " + obASTExpr.toString()); - } // 2.2 Convert ExprNode to RexNode - RexNode rnd = converter.convert(obExprNDesc); + rnd = converter.convert(obExprNDesc); // 2.3 Determine the index of ob expr in child schema // NOTE: Optiq can not take compound exprs in OB without it being @@ -13318,11 +13327,11 @@ private RelNode genOBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException fieldIndex = srcRelRecordSz + newVCLst.size(); newVCLst.add(rnd); vcASTTypePairs.add(new Pair((ASTNode) obASTExpr.getChild(0), - obExprNDesc.getTypeInfo())); + obExprNDesc.getTypeInfo())); } // 2.4 Determine the Direction of order by - RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING; + org.eigenbase.rel.RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING; if (obASTExpr.getType() == HiveParser.TOK_TABSORTCOLNAMEASC) { order = RelFieldCollation.Direction.ASCENDING; } @@ -13439,7 +13448,7 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept ExprNodeDesc exp = genExprNodeDesc(oExpr.getExpression(), inputRR, tcCtx); RexNode ordExp = converter.convert(exp); Set flags = new HashSet(); - if (oExpr.getOrder() == PTFInvocationSpec.Order.DESC) + if (oExpr.getOrder() == org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.DESC) flags.add(SqlKind.DESCENDING); oKeys.add(new RexFieldCollation(ordExp, flags)); } @@ -13457,12 +13466,13 @@ RexWindowBound getBound(BoundarySpec bs, RexNodeConverter converter) { SqlNode amt = bs.getAmt() == 0 ? null : SqlLiteral.createExactNumeric( String.valueOf(bs.getAmt()), new SqlParserPos(2, 2)); RexNode amtLiteral = null; + SqlCall sc = null; + RexNode rn = null; if (amt != null) - amtLiteral = cluster.getRexBuilder().makeLiteral(bs.getAmt(), + amtLiteral = cluster.getRexBuilder().makeLiteral(new Integer(bs.getAmt()), cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), true); - SqlCall sc = null; switch (bs.getDirection()) { case PRECEDING: if (amt == null) { @@ -13495,6 +13505,7 @@ RexWindowBound getBound(BoundarySpec bs, RexNodeConverter converter) { } int getWindowSpecIndx(ASTNode wndAST) { + int wndASTIndx = -1; int wi = wndAST.getChildCount() - 1; if (wi <= 0 || (wndAST.getChild(wi).getType() != HiveParser.TOK_WINDOWSPEC)) { wi = -1; @@ -13518,8 +13529,8 @@ int getWindowSpecIndx(ASTNode wndAST) { this.relToHiveRR.get(srcRel)); // 3. Get Optiq Return type for Agg Fn - wHiveRetType = hiveAggInfo.returnType; - RelDataType optiqAggFnRetType = TypeConverter.convert(hiveAggInfo.returnType, + wHiveRetType = hiveAggInfo.m_returnType; + RelDataType optiqAggFnRetType = TypeConverter.convert(hiveAggInfo.m_returnType, this.cluster.getTypeFactory()); // 4. Convert Agg Fn args to Optiq @@ -13529,16 +13540,16 @@ int getWindowSpecIndx(ASTNode wndAST) { Builder optiqAggFnArgsBldr = ImmutableList. builder(); Builder optiqAggFnArgsTypeBldr = ImmutableList. builder(); RexNode rexNd = null; - for (int i = 0; i < hiveAggInfo.aggParams.size(); i++) { - optiqAggFnArgsBldr.add(converter.convert(hiveAggInfo.aggParams.get(i))); - optiqAggFnArgsTypeBldr.add(TypeConverter.convert(hiveAggInfo.aggParams.get(i) + for (int i = 0; i < hiveAggInfo.m_aggParams.size(); i++) { + optiqAggFnArgsBldr.add(converter.convert(hiveAggInfo.m_aggParams.get(i))); + optiqAggFnArgsTypeBldr.add(TypeConverter.convert(hiveAggInfo.m_aggParams.get(i) .getTypeInfo(), this.cluster.getTypeFactory())); } ImmutableList optiqAggFnArgs = optiqAggFnArgsBldr.build(); ImmutableList optiqAggFnArgsType = optiqAggFnArgsTypeBldr.build(); // 5. Get Optiq Agg Fn - final SqlAggFunction optiqAggFn = SqlFunctionConverter.getOptiqAggFn(hiveAggInfo.udfName, + final SqlAggFunction optiqAggFn = SqlFunctionConverter.getOptiqAggFn(hiveAggInfo.m_udfName, optiqAggFnArgsType, optiqAggFnRetType); // 6. Translate Window spec @@ -13548,8 +13559,8 @@ int getWindowSpecIndx(ASTNode wndAST) { List orderKeys = getOrderKeys(wndSpec.getOrder(), converter, inputRR); RexWindowBound upperBound = getBound(wndSpec.windowFrame.start, converter); RexWindowBound lowerBound = getBound(wndSpec.windowFrame.end, converter); - boolean isRows = ((wndSpec.windowFrame.start instanceof RangeBoundarySpec) || - (wndSpec.windowFrame.end instanceof RangeBoundarySpec)); + boolean isRows = ((wndSpec.windowFrame.start instanceof RangeBoundarySpec) || (wndSpec.windowFrame.end instanceof RangeBoundarySpec)) ? true + : false; w = cluster.getRexBuilder().makeOver(optiqAggFnRetType, optiqAggFn, optiqAggFnArgs, partitionKeys, ImmutableList. copyOf(orderKeys), lowerBound, @@ -13565,9 +13576,8 @@ int getWindowSpecIndx(ASTNode wndAST) { private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticException { RelNode selOpForWindow = null; QBParseInfo qbp = getQBParseInfo(qb); - WindowingSpec wSpec = qb.getAllWindowingSpecs().isEmpty() - ? null - : qb.getAllWindowingSpecs().values().iterator().next(); + WindowingSpec wSpec = (!qb.getAllWindowingSpecs().isEmpty()) ? qb.getAllWindowingSpecs() + .values().iterator().next() : null; if (wSpec != null) { // 1. Get valid Window Function Spec @@ -13581,13 +13591,13 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticExce HiveOptiqUtil.getProjsFromBelowAsInputRef(srcRel)); // 3. Construct new Row Resolver with everything from below. - RowResolver outRwsch = new RowResolver(); - RowResolver.add(outRwsch, inputRR, 0); + RowResolver out_rwsch = new RowResolver(); + RowResolver.add(out_rwsch, inputRR, 0); // 4. Walk through Window Expressions & Construct RexNodes for those, // Update out_rwsch for (WindowExpressionSpec wExprSpec : windowExpressions) { - if (outRwsch.getExpression(wExprSpec.getExpression()) == null) { + if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) { Pair wtp = genWindowingProj(qb, wExprSpec, srcRel); projsForWindowSelOp.add(wtp.getKey()); @@ -13596,26 +13606,26 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticExce getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); String colAlias = wExprSpec.getAlias(); if (false) { - outRwsch.checkColumn(null, wExprSpec.getAlias()); - outRwsch.put(null, wExprSpec.getAlias(), oColInfo); + out_rwsch.checkColumn(null, wExprSpec.getAlias()); + out_rwsch.put(null, wExprSpec.getAlias(), oColInfo); } else { - outRwsch.putExpression(wExprSpec.getExpression(), oColInfo); + out_rwsch.putExpression(wExprSpec.getExpression(), oColInfo); } } } - selOpForWindow = genSelectRelNode(projsForWindowSelOp, outRwsch, srcRel); + selOpForWindow = genSelectRelNode(projsForWindowSelOp, out_rwsch, srcRel); } } return selOpForWindow; } - private RelNode genSelectRelNode(List optiqColLst, RowResolver outRwsch, + private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsch, RelNode srcRel) throws OptiqSemanticException { // 1. Build Column Names // TODO: Should this be external names - List columnNames = new ArrayList(); + ArrayList columnNames = new ArrayList(); for (int i = 0; i < optiqColLst.size(); i++) { columnNames.add(getColumnInternalName(i)); } @@ -13637,8 +13647,8 @@ public String apply(String hName) { HiveRel selRel = HiveProjectRel.create(srcRel, optiqColLst, oFieldNames); // 4. Keep track of colname-to-posmap && RR for new select - this.relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(outRwsch, selRel)); - this.relToHiveRR.put(selRel, outRwsch); + this.relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(out_rwsch, selRel)); + this.relToHiveRR.put(selRel, out_rwsch); return selRel; } @@ -13655,7 +13665,8 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep RelNode selForWindow = genSelectForWindowing(qb, srcRel); srcRel = (selForWindow == null) ? srcRel : selForWindow; - ArrayList colList = new ArrayList(); + boolean subQuery; + ArrayList col_list = new ArrayList(); ArrayList> windowingRexNodes = new ArrayList>(); // 1. Get Select Expression List @@ -13664,16 +13675,18 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep ASTNode selExprList = qbp.getSelForClause(selClauseName); // 2.Row resolvers for input, output - RowResolver outRwsch = new RowResolver(); + RowResolver out_rwsch = new RowResolver(); ASTNode trfm = null; - Integer pos = 0; + Integer pos = Integer.valueOf(0); RowResolver inputRR = this.relToHiveRR.get(srcRel); // 3. Query Hints // TODO: Handle Query Hints; currently we ignore them - boolean hintPresent = selExprList.getChild(0).getType() == HiveParser.TOK_HINTLIST; + boolean selectStar = false; + int posn = 0; + boolean hintPresent = (selExprList.getChild(0).getType() == HiveParser.TOK_HINTLIST); if (hintPresent) { - String hint = ctx.getTokenRewriteStream(). + String hint = SemanticAnalyzer.this.ctx.getTokenRewriteStream(). toString( selExprList.getChild(0).getTokenStartIndex(), selExprList.getChild(0).getTokenStopIndex()); @@ -13684,46 +13697,45 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep } // 4. Determine if select corresponds to a subquery - boolean subQuery = qb.getParseInfo().getIsSubQ(); + subQuery = qb.getParseInfo().getIsSubQ(); // 4. Bailout if select involves Transform - int posn = 0; - boolean isInTransform = - selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM; + boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { - String msg = "SELECT TRANSFORM is currently not supported in CBO," - + " turn off cbo to use TRANSFORM."; + String msg = String.format("SELECT TRANSFORM is currently not supported in CBO," + + " turn off cbo to use TRANSFORM."); LOG.debug(msg); throw new OptiqSemanticException(msg); } // 5. Bailout if select involves UDTF ASTNode udtfExpr = (ASTNode) selExprList.getChild(posn).getChild(0); + GenericUDTF genericUDTF = null; int udtfExprType = udtfExpr.getType(); if (udtfExprType == HiveParser.TOK_FUNCTION || udtfExprType == HiveParser.TOK_FUNCTIONSTAR) { String funcName = TypeCheckProcFactory.DefaultExprProcessor.getFunctionText(udtfExpr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); - GenericUDTF genericUDTF = null; if (fi != null) { genericUDTF = fi.getGenericUDTF(); } if (genericUDTF != null) { - String msg = "UDTF " + funcName + " is currently not supported in CBO," - + " turn off cbo to use UDTF " + funcName; + String msg = String.format("UDTF " + funcName + " is currently not supported in CBO," + + " turn off cbo to use UDTF " + funcName); LOG.debug(msg); throw new OptiqSemanticException(msg); } } // 6. Iterate over all expression (after SELECT) + ASTNode exprList = selExprList; + int startPosn = posn; int wndProjPos = 0; List tabAliasesForAllProjs = getTabAliases(inputRR); - boolean selectStar = false; - for (int i = posn; i < selExprList.getChildCount(); ++i) { + for (int i = startPosn; i < exprList.getChildCount(); ++i) { // 6.1 child can be EXPR AS ALIAS, or EXPR. - ASTNode child = (ASTNode) selExprList.getChild(i); - boolean hasAsClause = !isInTransform && (child.getChildCount() == 2); + ASTNode child = (ASTNode) exprList.getChild(i); + boolean hasAsClause = (!isInTransform) && (child.getChildCount() == 2); // 6.2 EXPR AS (ALIAS,...) parses, but is only allowed for UDTF's // This check is not needed and invalid when there is a transform b/c @@ -13734,18 +13746,22 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep ErrorMsg.INVALID_AS.getMsg())); } + ASTNode expr; + String tabAlias; + String colAlias; + // 6.3 Get rid of TOK_SELEXPR - ASTNode expr = (ASTNode) child.getChild(0); + expr = (ASTNode) child.getChild(0); String[] colRef = getColAlias(child, autogenColAliasPrfxLbl, inputRR, autogenColAliasPrfxIncludeFuncName, i); - String tabAlias = colRef[0]; - String colAlias = colRef[1]; + tabAlias = colRef[0]; + colAlias = colRef[1]; // 6.4 Build ExprNode corresponding to colums if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)) - .toLowerCase(), expr, colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, + .toLowerCase(), expr, col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause @@ -13755,7 +13771,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, - colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, subQuery); + col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText() @@ -13766,7 +13782,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr, - colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, subQuery); + col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -13774,44 +13790,45 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep tcCtx.setAllowStatefulFunctions(true); ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx); String recommended = recommendName(exp, colAlias); - if (recommended != null && outRwsch.get(null, recommended) == null) { + if (recommended != null && out_rwsch.get(null, recommended) == null) { colAlias = recommended; } - colList.add(exp); + col_list.add(exp); if (subQuery) { - outRwsch.checkColumn(tabAlias, colAlias); + out_rwsch.checkColumn(tabAlias, colAlias); } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); - colInfo.setSkewedCol( - (exp instanceof ExprNodeColumnDesc) && ((ExprNodeColumnDesc) exp).isSkewedCol()); - outRwsch.put(tabAlias, colAlias, colInfo); + colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) ? ((ExprNodeColumnDesc) exp) + .isSkewedCol() : false); + out_rwsch.put(tabAlias, colAlias, colInfo); if (exp instanceof ExprNodeColumnDesc) { ExprNodeColumnDesc colExp = (ExprNodeColumnDesc) exp; String[] altMapping = inputRR.getAlternateMappings(colExp.getColumn()); if (altMapping != null) { - outRwsch.put(altMapping[0], altMapping[1], colInfo); + out_rwsch.put(altMapping[0], altMapping[1], colInfo); } } - pos = pos.intValue() + 1; + pos = Integer.valueOf(pos.intValue() + 1); } } - selectStar = selectStar && selExprList.getChildCount() == posn + 1; + selectStar = selectStar && exprList.getChildCount() == posn + 1; // 7. Convert Hive projections to Optiq List optiqColLst = new ArrayList(); RexNodeConverter rexNodeConv = new RexNodeConverter(cluster, srcRel.getRowType(), - buildHiveColNameToInputPosMap(colList, inputRR), 0, false); - for (ExprNodeDesc colExpr : colList) { + buildHiveColNameToInputPosMap(col_list, inputRR), 0, false); + for (ExprNodeDesc colExpr : col_list) { optiqColLst.add(rexNodeConv.convert(colExpr)); } // 8. Build Optiq Rel + RelNode selRel = genSelectRelNode(optiqColLst, out_rwsch, srcRel); - return genSelectRelNode(optiqColLst, outRwsch, srcRel); + return selRel; } private RelNode genLogicalPlan(QBExpr qbexpr) throws SemanticException { @@ -13829,16 +13846,23 @@ private RelNode genLogicalPlan(QBExpr qbexpr) throws SemanticException { } private RelNode genLogicalPlan(QB qb) throws SemanticException { + RelNode srcRel = null; + RelNode filterRel = null; + RelNode gbRel = null; + RelNode gbHavingRel = null; RelNode havingRel = null; - RelNode rootRel = null; + RelNode selectRel = null; + RelNode obRel = null; + RelNode limitRel = null; + RelNode rootRel = null; // First generate all the opInfos for the elements in the from clause Map aliasToRel = new HashMap(); // 0. Check if we can handle the query // This check is needed here because of SubQuery if (!canHandleQuery(qb, false)) { - String msg = "CBO Can not handle Sub Query"; + String msg = String.format("CBO Can not handle Sub Query"); LOG.debug(msg); throw new OptiqSemanticException(msg); } @@ -13873,7 +13897,6 @@ private RelNode genLogicalPlan(QB qb) throws SemanticException { } // 1.3 process join - RelNode srcRel = null; if (qb.getParseInfo().getJoinExpr() != null) { srcRel = genJoinLogicalPlan(qb.getParseInfo().getJoinExpr(), aliasToRel); } else { @@ -13882,27 +13905,27 @@ private RelNode genLogicalPlan(QB qb) throws SemanticException { } // 2. Build Rel for where Clause - RelNode filterRel = genFilterLogicalPlan(qb, srcRel, aliasToRel, false); + filterRel = genFilterLogicalPlan(qb, srcRel, aliasToRel, false); srcRel = (filterRel == null) ? srcRel : filterRel; // 3. Build Rel for GB Clause - RelNode gbRel = genGBLogicalPlan(qb, srcRel); + gbRel = genGBLogicalPlan(qb, srcRel); srcRel = (gbRel == null) ? srcRel : gbRel; // 4. Build Rel for GB Having Clause - RelNode gbHavingRel = genGBHavingLogicalPlan(qb, srcRel, aliasToRel); + gbHavingRel = genGBHavingLogicalPlan(qb, srcRel, aliasToRel); srcRel = (gbHavingRel == null) ? srcRel : gbHavingRel; // 5. Build Rel for Select Clause - RelNode selectRel = genSelectLogicalPlan(qb, srcRel); + selectRel = genSelectLogicalPlan(qb, srcRel); srcRel = (selectRel == null) ? srcRel : selectRel; // 6. Build Rel for OB Clause - RelNode obRel = genOBLogicalPlan(qb, srcRel); + obRel = genOBLogicalPlan(qb, srcRel); srcRel = (obRel == null) ? srcRel : obRel; // 7. Build Rel for Limit Clause - RelNode limitRel = genLimitLogicalPlan(qb, srcRel); + limitRel = genLimitLogicalPlan(qb, srcRel); srcRel = (limitRel == null) ? srcRel : limitRel; // 8. Incase this QB corresponds to subquery then modify its RR to point @@ -13915,7 +13938,7 @@ private RelNode genLogicalPlan(QB qb) throws SemanticException { for (ColumnInfo colInfo : rr.getColumnInfos()) { String name = colInfo.getInternalName(); String[] tmp = rr.reverseLookup(name); - if (tmp[0] != null && tmp[0].isEmpty() || tmp[1] == null) { + if ("".equals(tmp[0]) || tmp[1] == null) { // ast expression is not a valid column name for table tmp[1] = colInfo.getInternalName(); } @@ -13997,8 +14020,8 @@ public Object post(Object t) { }; new TreeVisitor(ParseDriver.adaptor).visit(havingExpr, action); - if (!aliasReferences.isEmpty()) { - String havingClause = ctx + if (aliasReferences.size() > 0) { + String havingClause = SemanticAnalyzer.this.ctx .getTokenRewriteStream().toString( havingExpr.getTokenStartIndex(), havingExpr.getTokenStopIndex()); @@ -14024,14 +14047,15 @@ public Object post(Object t) { } private ImmutableMap buildHiveColNameToInputPosMap( - List colList, RowResolver inputRR) { + List col_list, RowResolver inputRR) { // Build a map of Hive column Names (ExprNodeColumnDesc Name) // to the positions of those projections in the input Map hashCodeTocolumnDescMap = new HashMap(); - ExprNodeDescUtils.getExprNodeColumnDesc(colList, hashCodeTocolumnDescMap); + ExprNodeDescUtils.getExprNodeColumnDesc(col_list, hashCodeTocolumnDescMap); ImmutableMap.Builder hiveColNameToInputPosMapBuilder = new ImmutableMap.Builder(); + String exprNodecolName; for (ExprNodeDesc exprDesc : hashCodeTocolumnDescMap.values()) { - String exprNodecolName = ((ExprNodeColumnDesc) exprDesc).getColumn(); + exprNodecolName = ((ExprNodeColumnDesc) exprDesc).getColumn(); hiveColNameToInputPosMapBuilder.put(exprNodecolName, inputRR.getPosition(exprNodecolName)); } @@ -14041,8 +14065,8 @@ public Object post(Object t) { private QBParseInfo getQBParseInfo(QB qb) throws OptiqSemanticException { QBParseInfo qbp = qb.getParseInfo(); if (qbp.getClauseNames().size() > 1) { - String msg = "Multi Insert is currently not supported in CBO," - + " turn off cbo to use Multi Insert."; + String msg = String.format("Multi Insert is currently not supported in CBO," + + " turn off cbo to use Multi Insert."); LOG.debug(msg); throw new OptiqSemanticException(msg); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 09a079c..e065983 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -118,7 +118,7 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) RowResolver input = ctx.getInputRR(); ExprNodeDesc desc = null; - if ((ctx == null) || (input == null) || !ctx.getAllowGBExprElimination()) { + if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) { return null; } @@ -769,11 +769,10 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString, isList); } else if (funcText.equals("[")) { // "[]" : LSQUARE/INDEX Expression - if (!ctx.getallowIndexExpr()) { + if (!ctx.getallowIndexExpr()) throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr)); - } - assert children.size() == 2; + assert (children.size() == 2); // Check whether this is a list or a map TypeInfo myt = children.get(0).getTypeInfo(); @@ -1030,10 +1029,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, * return null; */ if (windowingTokens.contains(expr.getType())) { - if (!ctx.getallowWindowing()) { + if (!ctx.getallowWindowing()) throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); - } + ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); return null; } @@ -1043,10 +1041,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (expr.getType() == HiveParser.TOK_ALLCOLREF) { - if (!ctx.getallowAllColRef()) { + if (!ctx.getallowAllColRef()) throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN.getMsg("All column reference is not supported in the context"))); - } + ErrorMsg.INVALID_COLUMN + .getMsg("All column reference is not supported in the context"))); RowResolver input = ctx.getInputRR(); ExprNodeColumnListDesc columnList = new ExprNodeColumnListDesc(); @@ -1119,10 +1117,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) { - if (!ctx.getallowFunctionStar()) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN.getMsg(".* reference is not supported in the context"))); - } + if (!ctx.getallowFunctionStar()) + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, + ErrorMsg.INVALID_COLUMN + .getMsg(".* reference is not supported in the context"))); RowResolver input = ctx.getInputRR(); for (ColumnInfo colInfo : input.getColumnInfos()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index 17d01f1..8a41577 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -94,8 +94,7 @@ public String getExprString() { return "null"; } - if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) || - typeInfo instanceof BaseCharTypeInfo) { + if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) || typeInfo instanceof BaseCharTypeInfo) { return "'" + value.toString() + "'"; } else if (typeInfo.getTypeName().equals(serdeConstants.BINARY_TYPE_NAME)) { byte[] bytes = (byte[]) value; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index 2dc69ec..75372cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -372,36 +372,42 @@ private static ExprNodeConstantDesc foldConstant(ExprNodeGenericFuncDesc func) { } catch (Exception e) { return null; } - } + } - public static void getExprNodeColumnDesc(List exprDescList, - Map hashCodeTocolumnDescMap) { - for (ExprNodeDesc exprNodeDesc : exprDescList) { - getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); - } - } + public static void getExprNodeColumnDesc(List exprDescList, + Map hashCodeTocolumnDescMap) { + for (ExprNodeDesc exprNodeDesc : exprDescList) { + getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); + } + } - /** - * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. - * - * @param hashCodeTocolumnDescMap - * Assumption: If two ExprNodeColumnDesc have same hash code then - * they are logically referring to same projection - */ - public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, - Map hashCodeTocolumnDescMap) { - if (exprDesc instanceof ExprNodeColumnDesc) { - hashCodeTocolumnDescMap.put(exprDesc.hashCode(), exprDesc); - } else if (exprDesc instanceof ExprNodeColumnListDesc) { - for (ExprNodeDesc child : exprDesc.getChildren()) { - getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); - } - } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { - for (ExprNodeDesc child : exprDesc.getChildren()) { - getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); - } - } else if (exprDesc instanceof ExprNodeFieldDesc) { - getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), hashCodeTocolumnDescMap); - } - } + /** + * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. + * + * @param exprDesc + * @param hashCodeTocolumnDescMap + * Assumption: If two ExprNodeColumnDesc have same hash code then + * they are logically referring to same projection + */ + public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, + Map hashCodeTocolumnDescMap) { + if (exprDesc instanceof ExprNodeColumnDesc) { + hashCodeTocolumnDescMap.put( + ((ExprNodeColumnDesc) exprDesc).hashCode(), + ((ExprNodeColumnDesc) exprDesc)); + } else if (exprDesc instanceof ExprNodeColumnListDesc) { + for (ExprNodeDesc child : ((ExprNodeColumnListDesc) exprDesc) + .getChildren()) { + getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); + } + } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { + for (ExprNodeDesc child : ((ExprNodeGenericFuncDesc) exprDesc) + .getChildren()) { + getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); + } + } else if (exprDesc instanceof ExprNodeFieldDesc) { + getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), + hashCodeTocolumnDescMap); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java index 8165715..f76fc10 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java @@ -36,8 +36,7 @@ @Description(name = "from_utc_timestamp", value = "from_utc_timestamp(timestamp, string timezone) - " - + "Assumes given timestamp ist UTC and converts to given timezone" - + " (as of Hive 0.8.0)") + + "Assumes given timestamp ist UTC and converts to given timezone (as of Hive 0.8.0)") public class GenericUDFFromUtcTimestamp extends GenericUDF { static final Log LOG = LogFactory.getLog(GenericUDFFromUtcTimestamp.class);