diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 5dab171..440c51a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -51,18 +51,19 @@ private int noOfJoins = 0; private int noOfOuterJoins = 0; private boolean hasLateralViews; - + private boolean multiDestQuery; private boolean filterWithSubQuery; - + public boolean hasJoin() { - return (noOfJoins > 0); + return noOfJoins > 0; } public void incrementJoinCount(boolean outerJoin) { noOfJoins++; - if (outerJoin) + if (outerJoin) { noOfOuterJoins++; + } } public int getJoinCount() { @@ -204,7 +205,7 @@ public void clear() { noOfJoins = 0; noOfOuterJoins = 0; - + multiDestQuery = false; filterWithSubQuery = false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index 4632f08..cdf8ab8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -269,7 +269,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { } } // init aggregationParameterFields - ArrayList aggrs = conf.getAggregators(); + List aggrs = conf.getAggregators(); aggregationParameterFields = new ExprNodeEvaluator[aggrs.size()][]; aggregationParameterObjectInspectors = new ObjectInspector[aggrs.size()][]; aggregationParameterStandardObjectInspectors = new ObjectInspector[aggrs.size()][]; @@ -277,7 +277,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { aggregationIsDistinct = new boolean[aggrs.size()]; for (int i = 0; i < aggrs.size(); i++) { AggregationDesc aggr = aggrs.get(i); - ArrayList parameters = aggr.getParameters(); + List parameters = aggr.getParameters(); aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()]; aggregationParameterObjectInspectors[i] = new ObjectInspector[parameters .size()]; @@ -1155,14 +1155,14 @@ public void closeOp(boolean abort) throws HiveException { public List genColLists( HashMap, OpParseContext> opParseCtx) { List colLists = new ArrayList(); - ArrayList keys = conf.getKeys(); + List keys = conf.getKeys(); for (ExprNodeDesc key : keys) { colLists = Utilities.mergeUniqElems(colLists, key.getCols()); } - ArrayList aggrs = conf.getAggregators(); + List aggrs = conf.getAggregators(); for (AggregationDesc aggr : aggrs) { - ArrayList params = aggr.getParameters(); + List params = aggr.getParameters(); for (ExprNodeDesc param : params) { colLists = Utilities.mergeUniqElems(colLists, param.getCols()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 34f5823..2cf6387 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -1935,7 +1935,7 @@ static String getUndecoratedName(String hiveTypeName) { public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, boolean isReduce) throws HiveException { - ArrayList paramDescList = desc.getParameters(); + List paramDescList = desc.getParameters(); VectorExpression[] vectorParams = new VectorExpression[paramDescList.size()]; for (int i = 0; i< paramDescList.size(); ++i) { @@ -1960,11 +1960,11 @@ public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, b inputType == VectorExpressionDescriptor.ArgumentType.NONE) || (aggDef.getType().isSameTypeOrFamily(inputType)))) { - if (aggDef.getMode() == GroupByDesc.Mode.HASH && isReduce) { - continue; - } else if (aggDef.getMode() == GroupByDesc.Mode.MERGEPARTIAL && !isReduce) { - continue; - } + if (aggDef.getMode() == GroupByDesc.Mode.HASH && isReduce) { + continue; + } else if (aggDef.getMode() == GroupByDesc.Mode.MERGEPARTIAL && !isReduce) { + continue; + } Class aggClass = aggDef.getAggClass(); try diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index ba28bc7..7e90cce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -130,14 +130,14 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; List colLists = new ArrayList(); GroupByDesc conf = op.getConf(); - ArrayList keys = conf.getKeys(); + List keys = conf.getKeys(); for (ExprNodeDesc key : keys) { colLists = Utilities.mergeUniqElems(colLists, key.getCols()); } - ArrayList aggrs = conf.getAggregators(); + List aggrs = conf.getAggregators(); for (AggregationDesc aggr : aggrs) { - ArrayList params = aggr.getParameters(); + List params = aggr.getParameters(); for (ExprNodeDesc param : params) { colLists = Utilities.mergeUniqElems(colLists, param.getCols()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java index e9e052f..d81437b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveDefaultRelMetadataProvider.java @@ -27,9 +27,7 @@ import org.eigenbase.rel.metadata.DefaultRelMetadataProvider; import org.eigenbase.rel.metadata.RelMetadataProvider; -public class HiveDefaultRelMetadataProvider { - private HiveDefaultRelMetadataProvider() { - } +public final class HiveDefaultRelMetadataProvider { public static final RelMetadataProvider INSTANCE = ChainedRelMetadataProvider.of(ImmutableList .of(HiveRelMdDistinctRowCount.SOURCE, @@ -37,4 +35,8 @@ private HiveDefaultRelMetadataProvider() { HiveRelMdRowCount.SOURCE, HiveRelMdUniqueKeys.SOURCE, new DefaultRelMetadataProvider())); + + private HiveDefaultRelMetadataProvider() { + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index 4bb99d9..58146f7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -52,8 +52,10 @@ /** * Generic utility functions needed for Optiq based Hive CBO. */ +public final class HiveOptiqUtil { -public class HiveOptiqUtil { + private HiveOptiqUtil() { + } /** * Get list of virtual columns from the given list of projections. @@ -78,7 +80,7 @@ public static boolean validateASTForCBO(ASTNode ast) { String astTree = ast.toStringTree(); // if any of following tokens are present in AST, bail out - String[] tokens = { "TOK_CHARSETLITERAL","TOK_TABLESPLITSAMPLE" }; + String[] tokens = {"TOK_CHARSETLITERAL", "TOK_TABLESPLITSAMPLE"}; for (String token : tokens) { if (astTree.contains(token)) { return false; @@ -88,14 +90,13 @@ public static boolean validateASTForCBO(ASTNode ast) { } public static List getProjsFromBelowAsInputRef(final RelNode rel) { - List projectList = Lists.transform(rel.getRowType().getFieldList(), + return Lists.transform(rel.getRowType().getFieldList(), new Function() { @Override public RexNode apply(RelDataTypeField field) { return rel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); } }); - return projectList; } public static List translateBitSetToProjIndx(BitSet projBitSet) { @@ -139,7 +140,6 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel RelNode leftRel = inputRels[0]; RelNode rightRel = inputRels[1]; RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder(); - RexNode outJoinCond = null; int origLeftInputSize = leftRel.getRowType().getFieldCount(); int origRightInputSize = rightRel.getRowType().getFieldCount(); @@ -182,6 +182,7 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel } } + RexNode outJoinCond = null; for (i = 0; i < origColEqConds.size(); i++) { Pair p = origColEqConds.get(i); RexNode leftKey = leftJoinKeys.get(i); @@ -250,12 +251,14 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel * */ public static class JoinPredicateInfo { - private final ImmutableList nonEquiJoinPredicateElements; - private final ImmutableList equiJoinPredicateElements; - private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; - private final ImmutableMap> mapOfProjIndxInJoinSchemaToLeafPInfo; + + private final ImmutableList nonEquiJoinPredicateElements; + private final ImmutableList equiJoinPredicateElements; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; + private final ImmutableMap> + mapOfProjIndxInJoinSchemaToLeafPInfo; public JoinPredicateInfo(List nonEquiJoinPredicateElements, List equiJoinPredicateElements, @@ -313,31 +316,28 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { } public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNode predicate) { - JoinPredicateInfo jpi = null; - JoinLeafPredicateInfo jlpi = null; List equiLPIList = new ArrayList(); List nonEquiLPIList = new ArrayList(); Set projsFromLeftPartOfJoinKeys = new HashSet(); Set projsFromRightPartOfJoinKeys = new HashSet(); Set projsFromRightPartOfJoinKeysInJoinSchema = new HashSet(); - Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); - Map> mapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>(); - List tmpJLPILst = null; + Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = + new HashMap>(); + Map> mapOfProjIndxInJoinSchemaToLeafPInfo = + new HashMap>(); int rightOffSet = j.getLeft().getRowType().getFieldCount(); - int projIndxInJoin; - List conjuctiveElements; // 1. Decompose Join condition to a number of leaf predicates // (conjuctive elements) - conjuctiveElements = RelOptUtil.conjunctions(predicate); + List conjuctiveElements = RelOptUtil.conjunctions(predicate); // 2. Walk through leaf predicates building up JoinLeafPredicateInfo for (RexNode ce : conjuctiveElements) { // 2.1 Construct JoinLeafPredicateInfo - jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); + JoinLeafPredicateInfo jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); // 2.2 Classify leaf predicate as Equi vs Non Equi - if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { + if (jlpi.comparisonType == SqlKind.EQUALS) { equiLPIList.add(jlpi); } else { nonEquiLPIList.add(jlpi); @@ -352,10 +352,12 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod // 2.4 Update Join Key to JoinLeafPredicateInfo map with keys // from left + List tmpJLPILst; for (Integer projIndx : jlpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) { tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndx); - if (tmpJLPILst == null) + if (tmpJLPILst == null) { tmpJLPILst = new ArrayList(); + } tmpJLPILst.add(jlpi); tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst); } @@ -363,10 +365,11 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod // 2.5 Update Join Key to JoinLeafPredicateInfo map with keys // from right for (Integer projIndx : jlpi.getProjsFromRightPartOfJoinKeysInChildSchema()) { - projIndxInJoin = projIndx + rightOffSet; + int projIndxInJoin = projIndx + rightOffSet; tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndxInJoin); - if (tmpJLPILst == null) + if (tmpJLPILst == null) { tmpJLPILst = new ArrayList(); + } tmpJLPILst.add(jlpi); tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndxInJoin, tmpJLPILst); } @@ -381,10 +384,9 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod } // 4. Construct JoinPredicateInfo - jpi = new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, - projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, - mapOfProjIndxInJoinSchemaToLeafPInfo); - return jpi; + return new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsFromLeftPartOfJoinKeys, + projsFromRightPartOfJoinKeys, projsFromRightPartOfJoinKeysInJoinSchema, + mapOfProjIndxInJoinSchemaToLeafPInfo); } } @@ -399,38 +401,38 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod * of equi join keys; the indexes are both in child and Join node schema.
*/ public static class JoinLeafPredicateInfo { - private final SqlKind m_comparisonType; - private final ImmutableList m_joinKeyExprsFromLeft; - private final ImmutableList m_joinKeyExprsFromRight; - private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; + private final SqlKind comparisonType; + private final ImmutableList joinKeyExprsFromLeft; + private final ImmutableList joinKeyExprsFromRight; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInJoinSchema) { - m_comparisonType = comparisonType; - m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); - m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); - m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + this.comparisonType = comparisonType; + this.joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); + this.joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); + this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); } public List getJoinKeyExprsFromLeft() { - return m_joinKeyExprsFromLeft; + return joinKeyExprsFromLeft; } public List getJoinKeyExprsFromRight() { - return m_joinKeyExprsFromRight; + return joinKeyExprsFromRight; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return projsFromLeftPartOfJoinKeysInChildSchema; } /** @@ -439,19 +441,18 @@ public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsF * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return m_projsFromRightPartOfJoinKeysInChildSchema; + return projsFromRightPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return m_projsFromRightPartOfJoinKeysInJoinSchema; + return projsFromRightPartOfJoinKeysInJoinSchema; } private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { - JoinLeafPredicateInfo jlpi = null; List filterNulls = new ArrayList(); List joinKeyExprsFromLeft = new ArrayList(); List joinKeyExprsFromRight = new ArrayList(); @@ -481,35 +482,29 @@ private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel } // 4. Construct JoinLeafPredicateInfo - jlpi = new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, - projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, - projsFromRightPartOfJoinKeysInJoinSchema); - return jlpi; + return new JoinLeafPredicateInfo(pe.getKind(), joinKeyExprsFromLeft, joinKeyExprsFromRight, + projsFromLeftPartOfJoinKeysInChildSchema, projsFromRightPartOfJoinKeysInChildSchema, + projsFromRightPartOfJoinKeysInJoinSchema); } } public static boolean limitRelNode(RelNode rel) { - if ((rel instanceof SortRel) && ((SortRel) rel).getCollation().getFieldCollations().isEmpty()) - return true; + return (rel instanceof SortRel) + && ((SortRel) rel).getCollation().getFieldCollations().isEmpty(); - return false; } public static boolean orderRelNode(RelNode rel) { - if ((rel instanceof SortRel) && !((SortRel) rel).getCollation().getFieldCollations().isEmpty()) - return true; + return (rel instanceof SortRel) + && !((SortRel) rel).getCollation().getFieldCollations().isEmpty(); - return false; } /** * Get top level select starting from root. Assumption here is root can only * be SortRel & ProjectRel. Also the top project should be at most 2 levels * below Sortrel; i.e SortRel(Limit)-SortRel(OB)-Select - * - * @param rootRel - * @return */ public static Pair getTopLevelSelect(final RelNode rootRel) { RelNode tmpRel = rootRel; @@ -525,6 +520,6 @@ public static boolean orderRelNode(RelNode rel) { tmpRel = tmpRel.getInput(0); } - return (new Pair(parentOforiginalProjRel, originalProjRel)); + return new Pair(parentOforiginalProjRel, originalProjRel); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java index d2b08fa..1ae5f4d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/OptiqSemanticException.java @@ -24,13 +24,11 @@ /** * Exception from SemanticAnalyzer. */ - public class OptiqSemanticException extends SemanticException { private static final long serialVersionUID = 1L; public OptiqSemanticException() { - super(); } public OptiqSemanticException(String message) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java index 080d27f..2f251fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java @@ -60,21 +60,21 @@ private final ImmutableMap hiveNonPartitionColsMap; private final ImmutableMap hivePartitionColsMap; private final int noOfProjs; - final HiveConf hiveConf; + private final HiveConf hiveConf; private double rowCount = -1; - Map hiveColStatsMap = new HashMap(); - PrunedPartitionList partitionList; - Map partitionCache; - AtomicInteger noColsMissingStats; - - protected static final Log LOG = LogFactory - .getLog(RelOptHiveTable.class - .getName()); - - public RelOptHiveTable(RelOptSchema optiqSchema, String qualifiedTblName, String tblAlias, RelDataType rowType, - Table hiveTblMetadata, List hiveNonPartitionCols, - List hivePartitionCols, HiveConf hconf, Map partitionCache, AtomicInteger noColsMissingStats) { + private final Map hiveColStatsMap = new HashMap(); + private PrunedPartitionList partitionList; + private final Map partitionCache; + private final AtomicInteger noColsMissingStats; + + private static final Log LOG = LogFactory.getLog(RelOptHiveTable.class.getName()); + + public RelOptHiveTable(RelOptSchema optiqSchema, String qualifiedTblName, String tblAlias, + RelDataType rowType, + Table hiveTblMetadata, List hiveNonPartitionCols, + List hivePartitionCols, HiveConf hconf, + Map partitionCache, AtomicInteger noColsMissingStats) { super(optiqSchema, qualifiedTblName, rowType); this.hiveTblMetadata = hiveTblMetadata; this.tblAlias = tblAlias; @@ -89,7 +89,7 @@ public RelOptHiveTable(RelOptSchema optiqSchema, String qualifiedTblName, String private static ImmutableMap getColInfoMap(List hiveCols, int startIndx) { - Builder bldr = ImmutableMap. builder(); + Builder bldr = ImmutableMap.builder(); int indx = startIndx; for (ColumnInfo ci : hiveCols) { @@ -101,7 +101,7 @@ public RelOptHiveTable(RelOptSchema optiqSchema, String qualifiedTblName, String } @Override - public boolean isKey(BitSet arg0) { + public boolean isKey(BitSet columns) { return false; } @@ -111,8 +111,8 @@ public RelNode toRel(ToRelContext context) { } @Override - public T unwrap(Class arg0) { - return arg0.isInstance(this) ? arg0.cast(this) : null; + public T unwrap(Class clazz) { + return clazz.isInstance(this) ? clazz.cast(this) : null; } @Override @@ -133,8 +133,9 @@ public double getRowCount() { } } - if (rowCount == -1) + if (rowCount == -1) { noColsMissingStats.getAndIncrement(); + } return rowCount; } @@ -150,21 +151,22 @@ public String getTableAlias() { // However in HIVE DB name can not appear in select list; in case of join // where table names differ only in DB name, Hive would require user // introducing explicit aliases for tbl. - if (tblAlias == null) + if (tblAlias == null) { return hiveTblMetadata.getTableName(); - else + } else { return tblAlias; + } } private String getColNamesForLogging(Set colLst) { - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); boolean firstEntry = true; for (String colName : colLst) { if (firstEntry) { sb.append(colName); firstEntry = false; } else { - sb.append(", " + colName); + sb.append(", ").append(colName); } } return sb.toString(); @@ -173,16 +175,21 @@ private String getColNamesForLogging(Set colLst) { public void computePartitionList(HiveConf conf, RexNode pruneNode) { try { - if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0 ) { + if (!hiveTblMetadata.isPartitioned() + || pruneNode == null + || InputFinder.bits(pruneNode).isEmpty()) { // there is no predicate on partitioning column, we need all partitions in this case. - partitionList = PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); + partitionList = + PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); return; } // We have valid pruning expressions, only retrieve qualifying partitions - ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true)); + ExprNodeDesc pruneExpr = + pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), true)); - partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); + partitionList = + PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); } catch (HiveException he) { throw new RuntimeException(he); } @@ -196,9 +203,9 @@ private void updateColStats(Set projIndxLst) { Set colNamesFailedStats = new HashSet(); // 1. Separate required columns to Non Partition and Partition Cols - ColumnInfo tmp; for (Integer pi : projIndxLst) { if (hiveColStatsMap.get(pi) == null) { + ColumnInfo tmp; if ((tmp = hiveNonPartitionColsMap.get(pi)) != null) { nonPartColNamesThatRqrStats.add(tmp.getInternalName()); nonPartColIndxsThatRqrStats.add(pi); @@ -222,13 +229,13 @@ private void updateColStats(Set projIndxLst) { } // 2. Obtain Col Stats for Non Partition Cols - if (nonPartColNamesThatRqrStats.size() > 0) { + if (!nonPartColNamesThatRqrStats.isEmpty()) { List hiveColStats; if (!hiveTblMetadata.isPartitioned()) { // 2.1 Handle the case for unpartitioned table. hiveColStats = StatsUtils.getTableColumnStats(hiveTblMetadata, hiveNonPartitionCols, - nonPartColNamesThatRqrStats); + nonPartColNamesThatRqrStats); // 2.1.1 Record Column Names that we needed stats for but couldn't if (hiveColStats == null) { @@ -257,17 +264,18 @@ private void updateColStats(Set projIndxLst) { } colNamesFailedStats.clear(); } else { - Statistics stats = StatsUtils.collectStatistics(hiveConf, partitionList, - hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats, - nonPartColNamesThatRqrStats, true, true); + Statistics + stats = StatsUtils.collectStatistics(hiveConf, partitionList, hiveTblMetadata, + hiveNonPartitionCols, nonPartColNamesThatRqrStats, nonPartColNamesThatRqrStats, true, + true); rowCount = stats.getNumRows(); hiveColStats = new ArrayList(); for (String c : nonPartColNamesThatRqrStats) { ColStatistics cs = stats.getColumnStatisticsFromColName(c); - if (cs != null) { - hiveColStats.add(cs); - } else { + if (cs == null) { colNamesFailedStats.add(c); + } else { + hiveColStats.add(cs); } } } @@ -290,9 +298,10 @@ private void updateColStats(Set projIndxLst) { ColStatistics cStats = null; for (int i = 0; i < partColNamesThatRqrStats.size(); i++) { cStats = new ColStatistics(hiveTblMetadata.getTableName(), - partColNamesThatRqrStats.get(i), hivePartitionColsMap.get( - partColIndxsThatRqrStats.get(i)).getTypeName()); - cStats.setCountDistint(getDistinctCount(partitionList.getPartitions(),partColNamesThatRqrStats.get(i))); + partColNamesThatRqrStats.get(i), hivePartitionColsMap.get( + partColIndxsThatRqrStats.get(i)).getTypeName()); + cStats.setCountDistint( + getDistinctCount(partitionList.getPartitions(), partColNamesThatRqrStats.get(i))); hiveColStatsMap.put(partColIndxsThatRqrStats.get(i), cStats); } } @@ -316,14 +325,9 @@ private int getDistinctCount(Set partitions, String partColName) { } public List getColStat(List projIndxLst) { - ImmutableList.Builder colStatsBldr = ImmutableList. builder(); + ImmutableList.Builder colStatsBldr = ImmutableList.builder(); - if (projIndxLst != null) { - updateColStats(new HashSet(projIndxLst)); - for (Integer i : projIndxLst) { - colStatsBldr.add(hiveColStatsMap.get(i)); - } - } else { + if (projIndxLst == null) { List pILst = new ArrayList(); for (Integer i = 0; i < noOfProjs; i++) { pILst.add(i); @@ -332,6 +336,11 @@ private int getDistinctCount(Set partitions, String partColName) { for (Integer pi : pILst) { colStatsBldr.add(hiveColStatsMap.get(pi)); } + } else { + updateColStats(new HashSet(projIndxLst)); + for (Integer i : projIndxLst) { + colStatsBldr.add(hiveColStatsMap.get(i)); + } } return colStatsBldr.build(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java index 9b653d3..a3ca4f3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java @@ -1,19 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq; - import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; import org.eigenbase.rel.RelCollation; import org.eigenbase.rel.RelCollationImpl; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; -public class TraitsUtil { +public final class TraitsUtil { + + private TraitsUtil() { + + } + public static RelTraitSet getSortTraitSet(RelOptCluster cluster, RelTraitSet traitSet, - RelCollation collation) { + RelCollation collation) { return traitSet.plus(collation); } public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) { return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java index 72fe5d6..99d39b3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCost.java @@ -25,44 +25,44 @@ public class HiveCost implements RelOptCost { // ~ Static fields/initializers --------------------------------------------- - public static final HiveCost INFINITY = new HiveCost(Double.POSITIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.POSITIVE_INFINITY) { - @Override - public String toString() { - return "{inf}"; - } - }; - - public static final HiveCost HUGE = new HiveCost(Double.MAX_VALUE, Double.MAX_VALUE, - Double.MAX_VALUE) { - @Override - public String toString() { - return "{huge}"; - } - }; - - public static final HiveCost ZERO = new HiveCost(0.0, 0.0, 0.0) { - @Override - public String toString() { - return "{0}"; - } - }; - - public static final HiveCost TINY = new HiveCost(1.0, 1.0, 0.0) { - @Override - public String toString() { - return "{tiny}"; - } - }; + public static final HiveCost INFINITY = new HiveCost(Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY) { + @Override + public String toString() { + return "{inf}"; + } + }; + + public static final HiveCost HUGE = new HiveCost(Double.MAX_VALUE, Double.MAX_VALUE, + Double.MAX_VALUE) { + @Override + public String toString() { + return "{huge}"; + } + }; + + public static final HiveCost ZERO = new HiveCost(0.0, 0.0, 0.0) { + @Override + public String toString() { + return "{0}"; + } + }; + + public static final HiveCost TINY = new HiveCost(1.0, 1.0, 0.0) { + @Override + public String toString() { + return "{tiny}"; + } + }; public static final RelOptCostFactory FACTORY = new Factory(); // ~ Instance fields -------------------------------------------------------- - final double cpu; - final double io; - final double rowCount; + private final double cpu; + private final double io; + private final double rowCount; // ~ Constructors ----------------------------------------------------------- @@ -77,23 +77,27 @@ public String toString() { // ~ Methods ---------------------------------------------------------------- + @Override public double getCpu() { return cpu; } + @Override public boolean isInfinite() { - return (this == INFINITY) || (this.rowCount == Double.POSITIVE_INFINITY) - || (this.cpu == Double.POSITIVE_INFINITY) || (this.io == Double.POSITIVE_INFINITY); + return (this == INFINITY) || (rowCount == Double.POSITIVE_INFINITY) + || (cpu == Double.POSITIVE_INFINITY) || (io == Double.POSITIVE_INFINITY); } + @Override public double getIo() { return io; } // TODO: If two cost is equal, could we do any better than comparing // cardinality (may be some other heuristics to break the tie) + @Override public boolean isLe(RelOptCost other) { - return this == other || this.rowCount <= other.getRows(); + return this == other || rowCount <= other.getRows(); /* * if (((this.dCpu + this.dIo) < (other.getCpu() + other.getIo())) || * ((this.dCpu + this.dIo) == (other.getCpu() + other.getIo()) && this.dRows @@ -101,19 +105,22 @@ public boolean isLe(RelOptCost other) { */ } + @Override public boolean isLt(RelOptCost other) { - return this.rowCount < other.getRows(); + return rowCount < other.getRows(); /* * return isLe(other) && !equals(other); */ } + @Override public double getRows() { return rowCount; } + @Override public boolean equals(RelOptCost other) { - return (this == other) || ((this.rowCount) == (other.getRows())); + return (this == other) || (rowCount == other.getRows()); /* * //TODO: should we consider cardinality as well? return (this == other) || @@ -121,8 +128,9 @@ public boolean equals(RelOptCost other) { */ } + @Override public boolean isEqWithEpsilon(RelOptCost other) { - return (this == other) || (Math.abs((this.rowCount) - (other.getRows())) < RelOptUtil.EPSILON); + return (this == other) || (Math.abs(rowCount - other.getRows()) < RelOptUtil.EPSILON); // Turn this one once we do the Algorithm selection in CBO /* * return (this == other) || (Math.abs((this.dCpu + this.dIo) - @@ -130,15 +138,17 @@ public boolean isEqWithEpsilon(RelOptCost other) { */ } + @Override public RelOptCost minus(RelOptCost other) { if (this == INFINITY) { return this; } - return new HiveCost(this.rowCount - other.getRows(), this.cpu - other.getCpu(), this.io + return new HiveCost(rowCount - other.getRows(), cpu - other.getCpu(), io - other.getIo()); } + @Override public RelOptCost multiplyBy(double factor) { if (this == INFINITY) { return this; @@ -146,24 +156,25 @@ public RelOptCost multiplyBy(double factor) { return new HiveCost(rowCount * factor, cpu * factor, io * factor); } + @Override public double divideBy(RelOptCost cost) { // Compute the geometric average of the ratios of all of the factors // which are non-zero and finite. double d = 1; double n = 0; - if ((this.rowCount != 0) && !Double.isInfinite(this.rowCount) && (cost.getRows() != 0) + if ((rowCount != 0) && !Double.isInfinite(rowCount) && (cost.getRows() != 0) && !Double.isInfinite(cost.getRows())) { - d *= this.rowCount / cost.getRows(); + d *= rowCount / cost.getRows(); ++n; } - if ((this.cpu != 0) && !Double.isInfinite(this.cpu) && (cost.getCpu() != 0) + if ((cpu != 0) && !Double.isInfinite(cpu) && (cost.getCpu() != 0) && !Double.isInfinite(cost.getCpu())) { - d *= this.cpu / cost.getCpu(); + d *= cpu / cost.getCpu(); ++n; } - if ((this.io != 0) && !Double.isInfinite(this.io) && (cost.getIo() != 0) + if ((io != 0) && !Double.isInfinite(io) && (cost.getIo() != 0) && !Double.isInfinite(cost.getIo())) { - d *= this.io / cost.getIo(); + d *= io / cost.getIo(); ++n; } if (n == 0) { @@ -172,11 +183,12 @@ public double divideBy(RelOptCost cost) { return Math.pow(d, 1 / n); } + @Override public RelOptCost plus(RelOptCost other) { - if ((this == INFINITY) || (other.isInfinite())) { + if ((this == INFINITY) || other.isInfinite()) { return INFINITY; } - return new HiveCost(this.rowCount + other.getRows(), this.cpu + other.getCpu(), this.io + return new HiveCost(rowCount + other.getRows(), cpu + other.getCpu(), io + other.getIo()); } @@ -185,26 +197,29 @@ public String toString() { return "{" + rowCount + " rows, " + cpu + " cpu, " + io + " io}"; } - private static class Factory implements RelOptCostFactory { - private Factory() { - } + private static final class Factory implements RelOptCostFactory { + @Override public RelOptCost makeCost(double rowCount, double cpu, double io) { return new HiveCost(rowCount, cpu, io); } + @Override public RelOptCost makeHugeCost() { return HUGE; } + @Override public HiveCost makeInfiniteCost() { return INFINITY; } + @Override public HiveCost makeTinyCost() { return TINY; } + @Override public HiveCost makeZeroCost() { return ZERO; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java index 7436f12..f7694bd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveCostUtil.java @@ -22,15 +22,19 @@ import org.eigenbase.relopt.RelOptCost; // Use this once we have Join Algorithm selection -public class HiveCostUtil { - private static final double cpuCostInNanoSec = 1.0; - private static final double netCostInNanoSec = 150 * cpuCostInNanoSec; - private static final double localFSWriteCostInNanoSec = 4 * netCostInNanoSec; - private static final double localFSReadCostInNanoSec = 4 * netCostInNanoSec; - private static final double hDFSWriteCostInNanoSec = 10 * localFSWriteCostInNanoSec; +public final class HiveCostUtil { + + private static final double CPU_COST_IN_NANO_SEC = 1.0; + private static final double NET_COST_IN_NANO_SEC = 150 * CPU_COST_IN_NANO_SEC; + private static final double LOCAL_FS_WRITE_COST_IN_NANO_SEC = 4 * NET_COST_IN_NANO_SEC; + private static final double HDFS_WRITE_COST_IN_NANO_SEC = 10 * LOCAL_FS_WRITE_COST_IN_NANO_SEC; + private static final double LOCAL_FS_READ_COST_IN_NANO_SEC = 4 * NET_COST_IN_NANO_SEC; @SuppressWarnings("unused") -//Use this once we have Join Algorithm selection - private static final double hDFSReadCostInNanoSec = 1.5 * localFSReadCostInNanoSec; + //Use this once we have Join Algorithm selection + private static final double HDFS_READ_COST_IN_NANO_SEC = 1.5 * LOCAL_FS_READ_COST_IN_NANO_SEC; + + private HiveCostUtil() { + } public static RelOptCost computCardinalityBasedCost(HiveRel hr) { return new HiveCost(hr.getRows(), 0, 0); @@ -38,6 +42,7 @@ public static RelOptCost computCardinalityBasedCost(HiveRel hr) { public static HiveCost computeCost(HiveTableScanRel t) { double cardinality = t.getRows(); - return new HiveCost(cardinality, 0, hDFSWriteCostInNanoSec * cardinality * 0); + return new HiveCost(cardinality, 0, HDFS_WRITE_COST_IN_NANO_SEC * cardinality * 0); } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java index 15596bc..6ee1793 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.optimizer.optiq.cost; import org.eigenbase.rel.RelCollationTraitDef; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java index fc19895..f5a67cb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveCost; import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; -import org.eigenbase.rel.InvalidRelException; import org.eigenbase.rel.RelFactories.AggregateFactory; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.RelMetadataQuery; @@ -38,20 +37,14 @@ public static final HiveAggRelFactory HIVE_AGGR_REL_FACTORY = new HiveAggRelFactory(); public HiveAggregateRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, - BitSet groupSet, List aggCalls) throws InvalidRelException { + BitSet groupSet, List aggCalls) { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, groupSet, aggCalls); } @Override public AggregateRelBase copy(RelTraitSet traitSet, RelNode input, BitSet groupSet, - List aggCalls) { - try { - return new HiveAggregateRel(getCluster(), traitSet, input, groupSet, aggCalls); - } catch (InvalidRelException e) { - // Semantic error not possible. Must be a bug. Convert to - // internal error. - throw new AssertionError(e); - } + List aggCalls) { + return new HiveAggregateRel(getCluster(), traitSet, input, groupSet, aggCalls); } @Override @@ -74,11 +67,8 @@ public double getRows() { @Override public RelNode createAggregate(RelNode child, BitSet groupSet, List aggCalls) { - try { - return new HiveAggregateRel(child.getCluster(), child.getTraitSet(), child, groupSet, aggCalls); - } catch (InvalidRelException e) { - throw new RuntimeException(e); - } + return new HiveAggregateRel(child.getCluster(), child.getTraitSet(), child, groupSet, + aggCalls); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java index 8b85046..9ee24ce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveFilterRel.java @@ -32,7 +32,8 @@ public static final FilterFactory DEFAULT_FILTER_FACTORY = new HiveFilterFactoryImpl(); - public HiveFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { + public HiveFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, + RexNode condition) { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); } @@ -53,15 +54,13 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { /** * Implementation of {@link FilterFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel} - * . + * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel}. */ private static class HiveFilterFactoryImpl implements FilterFactory { @Override public RelNode createFilter(RelNode child, RexNode condition) { RelOptCluster cluster = child.getCluster(); - HiveFilterRel filter = new HiveFilterRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); - return filter; + return new HiveFilterRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java index 3d6aa84..70b2fe2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.cost.HiveCost; -import org.eigenbase.rel.InvalidRelException; import org.eigenbase.rel.JoinRelBase; import org.eigenbase.rel.JoinRelType; import org.eigenbase.rel.RelFactories.JoinFactory; @@ -61,21 +60,17 @@ public static HiveJoinRel getJoin(RelOptCluster cluster, RelNode left, RelNode right, RexNode condition, JoinRelType joinType, boolean leftSemiJoin) { - try { - Set variablesStopped = Collections.emptySet(); - return new HiveJoinRel(cluster, null, left, right, condition, joinType, variablesStopped, - JoinAlgorithm.NONE, null, leftSemiJoin); - } catch (InvalidRelException e) { - throw new RuntimeException(e); - } + Set variablesStopped = Collections.emptySet(); + return new HiveJoinRel(cluster, null, left, right, condition, joinType, variablesStopped, + JoinAlgorithm.NONE, null, leftSemiJoin); } protected HiveJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, - RexNode condition, JoinRelType joinType, Set variablesStopped, - JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin, boolean leftSemiJoin) - throws InvalidRelException { + RexNode condition, JoinRelType joinType, Set variablesStopped, + JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin, + boolean leftSemiJoin) { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), left, right, condition, joinType, - variablesStopped); + variablesStopped); this.joinAlgorithm = joinAlgo; this.leftSemiJoin = leftSemiJoin; } @@ -87,15 +82,9 @@ public void implement(Implementor implementor) { @Override public final HiveJoinRel copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { - try { - Set variablesStopped = Collections.emptySet(); - return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType, - variablesStopped, JoinAlgorithm.NONE, null, leftSemiJoin); - } catch (InvalidRelException e) { - // Semantic error not possible. Must be a bug. Convert to - // internal error. - throw new AssertionError(e); - } + Set variablesStopped = Collections.emptySet(); + return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType, + variablesStopped, JoinAlgorithm.NONE, null, leftSemiJoin); } public JoinAlgorithm getJoinAlgorithm() { @@ -124,7 +113,7 @@ public RelDataType deriveRowType() { if (leftSemiJoin) { return deriveJoinRowType(left.getRowType(), null, JoinRelType.INNER, getCluster().getTypeFactory(), null, - Collections. emptyList()); + Collections.emptyList()); } return super.deriveRowType(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java index f8755d0..0926b7e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java @@ -47,6 +47,7 @@ public HiveLimitRel copy(RelTraitSet traitSet, List newInputs) { return new HiveLimitRel(getCluster(), traitSet, sole(newInputs), offset, fetch); } + @Override public void implement(Implementor implementor) { } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java index 7b434ea..6413c27 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java @@ -48,7 +48,7 @@ public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl(); - private final List virtualCols; + private final List virtualCols; /** * Creates a HiveProjectRel. @@ -81,7 +81,7 @@ public HiveProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child * aliases of the expressions */ public static HiveProjectRel create(RelNode child, List exps, - List fieldNames) throws OptiqSemanticException{ + List fieldNames) throws OptiqSemanticException { RelOptCluster cluster = child.getCluster(); // 1 Ensure columnNames are unique - OPTIQ-411 @@ -90,14 +90,15 @@ public static HiveProjectRel create(RelNode child, List exps, throw new OptiqSemanticException(msg); } RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), exps, fieldNames); - return create(cluster, child, exps, rowType, Collections. emptyList()); + return create(cluster, child, exps, rowType, Collections.emptyList()); } /** * Creates a HiveProjectRel. */ - public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, - RelDataType rowType, final List collationList) { + public static HiveProjectRel create(RelOptCluster cluster, RelNode child, + List exps, + RelDataType rowType, final List collationList) { RelTraitSet traitSet = TraitsUtil.getDefaultTraitSet(cluster); return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -105,8 +106,9 @@ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, - RelDataType rowType, RelTraitSet traitSet, final List collationList) { + public static HiveProjectRel create(RelOptCluster cluster, RelNode child, + List exps, + RelDataType rowType, RelTraitSet traitSet, final List collationList) { return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -121,7 +123,7 @@ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List - * This method could optimize the result as {@link #permute} does, but does + * This method could optimize the result as {@link #getPermutation()} does, but does * not at present. * * @param rel @@ -137,7 +139,8 @@ public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List fieldNames) throws OptiqSemanticException { + public static RelNode projectMapping(RelNode rel, Mapping mapping, List fieldNames) + throws OptiqSemanticException { assert mapping.getMappingType().isSingleSource(); assert mapping.getMappingType().isMandatorySource(); @@ -154,8 +157,10 @@ public static RelNode projectMapping(RelNode rel, Mapping mapping, List int source = mapping.getSource(i); final RelDataTypeField sourceField = fields.get(source); outputNameList - .add(((fieldNames == null) || (fieldNames.size() <= i) || (fieldNames.get(i) == null)) ? sourceField - .getName() : fieldNames.get(i)); + .add(((fieldNames == null) || (fieldNames.size() <= i) || (fieldNames.get(i) == null)) + ? sourceField + .getName() + : fieldNames.get(i)); outputProjList.add(rexBuilder.makeInputRef(rel, source)); } @@ -184,8 +189,7 @@ public void implement(Implementor implementor) { /** * Implementation of {@link ProjectFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel} - * . + * {@link org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel}. */ private static class HiveProjectFactoryImpl implements ProjectFactory { @@ -193,12 +197,12 @@ public void implement(Implementor implementor) { public RelNode createProject(RelNode child, List childExprs, List fieldNames) { RelOptCluster cluster = child.getCluster(); - RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); - RelNode project = HiveProjectRel.create(cluster, child, - childExprs, rowType, - child.getTraitSet(), Collections. emptyList()); + RelDataType rowType = + RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); - return project; + return create(cluster, child, + childExprs, rowType, + child.getTraitSet(), Collections.emptyList()); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java index 4738c4a..cf854b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveRel.java @@ -21,10 +21,10 @@ import org.eigenbase.relopt.Convention; public interface HiveRel extends RelNode { - void implement(Implementor implementor); - /** Calling convention for relational operations that occur in Hive. */ - final Convention CONVENTION = new Convention.Impl("HIVE", HiveRel.class); + Convention CONVENTION = new Convention.Impl("HIVE", HiveRel.class); + + void implement(Implementor implementor); class Implementor { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java index f85363d..e328595 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java @@ -63,11 +63,11 @@ public RexNode getFetchExpr() { } public void setInputRefToCallMap(ImmutableMap refToCall) { - this.mapOfInputRefToRexCall = refToCall; + mapOfInputRefToRexCall = refToCall; } public Map getInputRefToCallMap() { - return this.mapOfInputRefToRexCall; + return mapOfInputRefToRexCall; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java index bd66459..e2c4edd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java @@ -50,8 +50,6 @@ * @param traitSet * Traits * @param table - * Table - * @param table * HiveDB table */ public HiveTableScanRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptHiveTable table, @@ -83,10 +81,11 @@ public void implement(Implementor implementor) { @Override public double getRows() { - return ((RelOptHiveTable) table).getRowCount(); + return table.getRowCount(); } public List getColStat(List projIndxLst) { return ((RelOptHiveTable) table).getColStat(projIndxLst); } -} \ No newline at end of file +} + diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java index d34fe95..063ddef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java @@ -49,7 +49,8 @@ public void implement(Implementor implementor) { @Override public RelNode createSetOp(SqlKind kind, List inputs, boolean all) { if (kind != SqlKind.UNION) { - throw new IllegalStateException("Expected to get Set operator of type Union. Found : " + kind); + throw new IllegalStateException( + "Expected to get Set operator of type Union. Found : " + kind); } return new HiveUnionRel(inputs.get(0).getCluster(), inputs.get(0).getTraitSet(), inputs); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java index ee19a6c..9f66f24 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePartitionPrunerRule.java @@ -29,7 +29,7 @@ public class HivePartitionPrunerRule extends RelOptRule { - HiveConf conf; + private final HiveConf conf; public HivePartitionPrunerRule(HiveConf conf) { super(operand(HiveFilterRel.class, operand(HiveTableScanRel.class, none()))); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java index da0f7a4..d30eb4b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java @@ -110,8 +110,8 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, } } - final List aboveFilters = filter != null ? RelOptUtil - .conjunctions(filter.getCondition()) : new ArrayList(); + final List aboveFilters = filter == null ? new ArrayList() : RelOptUtil + .conjunctions(filter.getCondition()); List leftFilters = new ArrayList(); List rightFilters = new ArrayList(); @@ -155,8 +155,9 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, break; } } - if (validHiveJoinFilter) + if (validHiveJoinFilter) { continue; + } } } aboveFilters.add(exp); @@ -166,7 +167,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, /* * if all pushed filters where put back then set filterPushed to false */ - if (leftFilters.size() == 0 && rightFilters.size() == 0 + if (leftFilters.isEmpty() && rightFilters.isEmpty() && joinFilters.size() == origJoinFiltersSz) { filterPushed = false; } @@ -202,7 +203,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, // containing its new join filters (if there are any) RexNode joinFilter; - if (joinFilters.size() == 0) { + if (joinFilters.isEmpty()) { // if nothing actually got pushed and there is nothing leftover, // then this rule is a no-op if (leftFilters.isEmpty() @@ -225,7 +226,7 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, /** * If the filter list passed in is non-empty, creates a FilterRel on top of - * the existing RelNode; otherwise, just returns the RelNode + * the existing RelNode; otherwise, just returns the RelNode. * * @param rexBuilder * rex builder @@ -268,7 +269,6 @@ private boolean isAlwaysTrue(RexNode predicate) { } private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { - boolean refersToBothSides = false; int joinNoOfProjects = j.getRowType().getFieldCount(); BitSet filterProjs = new BitSet(joinNoOfProjects); @@ -280,12 +280,12 @@ private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { InputFinder inputFinder = new InputFinder(filterProjs); filter.accept(inputFinder); - if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) + boolean refersToBothSides = false; + if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) { refersToBothSides = true; + } return refersToBothSides; } } -// End PushFilterPastJoinRule.java - diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java index a218eca..9c5ade0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/PartitionPruner.java @@ -39,16 +39,17 @@ import org.eigenbase.sql.fun.SqlStdOperatorTable; import org.eigenbase.util.Pair; -public class PartitionPruner { +public final class PartitionPruner { + + private PartitionPruner() { + + } /** * Breaks the predicate into 2 pieces. The first piece is the expressions that * only contain partition columns and can be used for Partition Pruning; the * second piece is the predicates that are left. - * - * @param cluster - * @param hiveTable - * @param predicate + * * @return a Pair of expressions, each of which maybe null. The 1st predicate * is expressions that only contain partition columns; the 2nd * predicate contains the remaining predicates. @@ -65,10 +66,10 @@ public static class ExtractPartPruningPredicate extends RexVisitorImpl { - final RelOptHiveTable hiveTable; - final RelDataType rType; - final Set partCols; - final RelOptCluster cluster; + private final RelOptHiveTable hiveTable; + private final RelDataType rType; + private final Set partCols; + private final RelOptCluster cluster; public ExtractPartPruningPredicate(RelOptCluster cluster, RelOptHiveTable hiveTable) { @@ -105,28 +106,27 @@ public RexNode visitCall(RexCall call) { } List args = new LinkedList(); - boolean argsPruned = false; GenericUDF hiveUDF = SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType()); - if (hiveUDF != null && - !FunctionRegistry.isDeterministic(hiveUDF)) { + if (hiveUDF != null && !FunctionRegistry.isDeterministic(hiveUDF)) { return null; } + boolean argsPruned = false; for (RexNode operand : call.operands) { RexNode n = operand.accept(this); - if (n != null) { - args.add(n); - } else { + if (n == null) { argsPruned = true; + } else { + args.add(n); } } if (call.getOperator() != SqlStdOperatorTable.AND) { return argsPruned ? null : call; } else { - if (args.size() == 0) { + if (args.isEmpty()) { return null; } else if (args.size() == 1) { return args.get(0); @@ -140,8 +140,8 @@ public RexNode visitCall(RexCall call) { public static class ExtractRemainingPredicate extends RexVisitorImpl { - List pruningPredicates; - final RelOptCluster cluster; + private List pruningPredicates; + private final RelOptCluster cluster; public ExtractRemainingPredicate(RelOptCluster cluster, RexNode partPruningExpr) { @@ -195,7 +195,7 @@ public RexNode visitCall(RexCall call) { } } - if (args.size() == 0) { + if (args.isEmpty()) { return null; } else if (args.size() == 1) { return args.get(0); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java index 214a29f..a95e9c5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java @@ -34,19 +34,20 @@ import org.eigenbase.sql.SqlKind; public class FilterSelectivityEstimator extends RexVisitorImpl { - private final RelNode m_childRel; - private final double m_childCardinality; + private final RelNode childRel; + private final double childCardinality; protected FilterSelectivityEstimator(RelNode childRel) { super(true); - m_childRel = childRel; - m_childCardinality = RelMetadataQuery.getRowCount(m_childRel); + this.childRel = childRel; + childCardinality = RelMetadataQuery.getRowCount(this.childRel); } public Double estimateSelectivity(RexNode predicate) { return predicate.accept(this); } + @Override public Double visitCall(RexCall call) { if (!deep) { return 1.0; @@ -57,7 +58,7 @@ public Double visitCall(RexCall call) { * because we have already accounted for these in * the Table row count. */ - if (isPartitionPredicate(call, m_childRel)) { + if (isPartitionPredicate(call, childRel)) { return 1.0; } @@ -65,33 +66,28 @@ public Double visitCall(RexCall call) { SqlKind op = call.getKind(); switch (op) { - case AND: { + case AND: selectivity = computeConjunctionSelectivity(call); break; - } - case OR: { + case OR: selectivity = computeDisjunctionSelectivity(call); break; - } - case NOT_EQUALS: { + case NOT_EQUALS: selectivity = computeNotEqualitySelectivity(call); break; - } case LESS_THAN_OR_EQUAL: case GREATER_THAN_OR_EQUAL: case LESS_THAN: - case GREATER_THAN: { + case GREATER_THAN: selectivity = ((double) 1 / (double) 3); break; - } - case IN: { + case IN: selectivity = ((double) 1 / ((double) call.operands.size())); break; - } default: selectivity = computeFunctionSelectivity(call); @@ -103,18 +99,15 @@ public Double visitCall(RexCall call) { /** * NDV of "f1(x, y, z) != f2(p, q, r)" -> * "(maxNDV(x,y,z,p,q,r) - 1)/maxNDV(x,y,z,p,q,r)". - *

- * - * @param call - * @return */ private Double computeNotEqualitySelectivity(RexCall call) { double tmpNDV = getMaxNDV(call); - if (tmpNDV > 1) + if (tmpNDV > 1) { return (tmpNDV - (double) 1) / tmpNDV; - else + } else { return 1.0; + } } /** @@ -122,9 +115,6 @@ private Double computeNotEqualitySelectivity(RexCall call) { *

* Note that >, >=, <, <=, = ... are considered generic functions and uses * this method to find their selectivity. - * - * @param call - * @return */ private Double computeFunctionSelectivity(RexCall call) { return 1 / getMaxNDV(call); @@ -137,49 +127,42 @@ private Double computeFunctionSelectivity(RexCall call) { *

* Note we compute m1. m2.. by applying selectivity of the disjunctive element * on the cardinality from child. - * - * @param call - * @return */ private Double computeDisjunctionSelectivity(RexCall call) { - Double tmpCardinality; - Double tmpSelectivity; double selectivity = 1; for (RexNode dje : call.getOperands()) { - tmpSelectivity = dje.accept(this); + Double tmpSelectivity = dje.accept(this); if (tmpSelectivity == null) { tmpSelectivity = 0.99; } - tmpCardinality = m_childCardinality * tmpSelectivity; + Double tmpCardinality = childCardinality * tmpSelectivity; - if (tmpCardinality > 1) - tmpSelectivity = (1 - tmpCardinality / m_childCardinality); - else + if (tmpCardinality > 1) { + tmpSelectivity = 1 - tmpCardinality / childCardinality; + } else { tmpSelectivity = 1.0; + } selectivity *= tmpSelectivity; } - if (selectivity > 1) - return (1 - selectivity); - else + if (selectivity > 1) { + return 1 - selectivity; + } else { return 1.0; + } } /** * Selectivity of conjunctive predicate -> (selectivity of conjunctive * element1) * (selectivity of conjunctive element2)... - * - * @param call - * @return */ private Double computeConjunctionSelectivity(RexCall call) { - Double tmpSelectivity; double selectivity = 1; for (RexNode cje : call.getOperands()) { - tmpSelectivity = cje.accept(this); + Double tmpSelectivity = cje.accept(this); if (tmpSelectivity != null) { selectivity *= tmpSelectivity; } @@ -189,23 +172,24 @@ private Double computeConjunctionSelectivity(RexCall call) { } private Double getMaxNDV(RexCall call) { - double tmpNDV; double maxNDV = 1.0; - InputReferencedVisitor irv; for (RexNode op : call.getOperands()) { + double tmpNDV; if (op instanceof RexInputRef) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(childRel, ((RexInputRef) op).getIndex()); - if (tmpNDV > maxNDV) + if (tmpNDV > maxNDV) { maxNDV = tmpNDV; + } } else { - irv = new InputReferencedVisitor(); + InputReferencedVisitor irv = new InputReferencedVisitor(); irv.apply(op); for (Integer childProjIndx : irv.inputPosReferenced) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, childProjIndx); - if (tmpNDV > maxNDV) + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(childRel, childProjIndx); + if (tmpNDV > maxNDV) { maxNDV = tmpNDV; + } } } } @@ -214,14 +198,14 @@ private Double getMaxNDV(RexCall call) { } private boolean isPartitionPredicate(RexNode expr, RelNode r) { - if ( r instanceof ProjectRelBase ) { + if (r instanceof ProjectRelBase) { expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r); return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild()); - } else if ( r instanceof FilterRelBase ) { + } else if (r instanceof FilterRelBase) { return isPartitionPredicate(expr, ((FilterRelBase) r).getChild()); - } else if ( r instanceof HiveTableScanRel ) { + } else if (r instanceof HiveTableScanRel) { RelOptHiveTable table = (RelOptHiveTable) - ((HiveTableScanRel)r).getTable(); + r.getTable(); BitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java index 4be57b1..279b525 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount { +public final class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount { private static final HiveRelMdDistinctRowCount INSTANCE = new HiveRelMdDistinctRowCount(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java index 949eb19..da35b8d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hive.ql.optimizer.optiq.stats; import java.util.ArrayList; @@ -52,20 +51,19 @@ public class HiveRelMdRowCount extends RelMdRowCount { - protected static final Log LOG = LogFactory.getLog(HiveRelMdRowCount.class.getName()); + private static final Log LOG = LogFactory.getLog(HiveRelMdRowCount.class.getName()); public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider .reflectiveSource(BuiltinMethod.ROW_COUNT.method, new HiveRelMdRowCount()); protected HiveRelMdRowCount() { - super(); } public Double getRowCount(JoinRelBase join) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(join); if (pkfk != null) { - double selectivity = (pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor); + double selectivity = pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor; selectivity = Math.min(1.0, selectivity); if (LOG.isDebugEnabled()) { LOG.debug("Identified Primary - Foreign Key relation:"); @@ -80,7 +78,7 @@ public Double getRowCount(JoinRelBase join) { public Double getRowCount(SemiJoinRel rel) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(rel); if (pkfk != null) { - double selectivity = (pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor); + double selectivity = pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor; selectivity = Math.min(1.0, selectivity); if (LOG.isDebugEnabled()) { LOG.debug("Identified Primary - Foreign Key relation:"); @@ -113,8 +111,8 @@ public Double getRowCount(SemiJoinRel rel) { public String toString() { return String.format( - "Primary - Foreign Key join:\n\tfkSide = %d\n\tFKInfo:%s\n" + - "\tPKInfo:%s\n\tisPKSideSimple:%s\n\tNDV Scaling Factor:%.2f\n", + "Primary - Foreign Key join:%n\tfkSide = %d%n\tFKInfo:%s%n" + + "\tPKInfo:%s%n\tisPKSideSimple:%s%n\tNDV Scaling Factor:%.2f%n", fkSide, fkInfo, pkInfo, @@ -138,13 +136,14 @@ public String toString() { static class PKSideInfo extends FKSideInfo { public final double selectivity; - public PKSideInfo(double rowCount, double distinctCount, double selectivity) { + PKSideInfo(double rowCount, double distinctCount, double selectivity) { super(rowCount, distinctCount); this.selectivity = selectivity; } public String toString() { - return String.format("PKInfo(rowCount=%.2f,ndv=%.2f,selectivity=%.2f)", rowCount, distinctCount,selectivity); + return String.format("PKInfo(rowCount=%.2f,ndv=%.2f,selectivity=%.2f)", rowCount, + distinctCount, selectivity); } } @@ -225,47 +224,51 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { double leftRowCount = RelMetadataQuery.getRowCount(left); double rightRowCount = RelMetadataQuery.getRowCount(right); - if (leftIsKey && rightIsKey) { - if (rightRowCount < leftRowCount) { - leftIsKey = false; - } + if (leftIsKey && rightIsKey && rightRowCount < leftRowCount) { + leftIsKey = false; } - int pkSide = leftIsKey ? 0 : rightIsKey ? 1 : -1; - - boolean isPKSideSimpleTree = pkSide != -1 ? - IsSimpleTreeOnJoinKey.check( - pkSide == 0 ? left : right, - pkSide == 0 ? leftColIdx : rightColIdx) : false; - - double leftNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(left, lBitSet, leftPred) : -1; - double rightNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(right, rBitSet, rightPred) : -1; - - /* - * If the ndv of the PK - FK side don't match, and the PK side is a filter - * on the Key column then scale the NDV on the FK side. - * - * As described by Peter Boncz: http://databasearchitects.blogspot.com/ - * in such cases we can be off by a large margin in the Join cardinality - * estimate. The e.g. he provides is on the join of StoreSales and DateDim - * on the TPCDS dataset. Since the DateDim is populated for 20 years into - * the future, while the StoreSales only has 5 years worth of data, there - * are 40 times fewer distinct dates in StoreSales. - * - * In general it is hard to infer the range for the foreign key on an - * arbitrary expression. For e.g. the NDV for DayofWeek is the same - * irrespective of NDV on the number of unique days, whereas the - * NDV of Quarters has the same ratio as the NDV on the keys. - * - * But for expressions that apply only on columns that have the same NDV - * as the key (implying that they are alternate keys) we can apply the - * ratio. So in the case of StoreSales - DateDim joins for predicate on the - * d_date column we can apply the scaling factor. - */ - double ndvScalingFactor = 1.0; - if ( isPKSideSimpleTree ) { - ndvScalingFactor = pkSide == 0 ? leftNDV/rightNDV : rightNDV / leftNDV; - } + int pkSide; + if (rightIsKey) { + pkSide = leftIsKey ? 0 : 1; + } else { + pkSide = leftIsKey ? 0 : -1; + } + + boolean isPKSideSimpleTree = pkSide != -1 && IsSimpleTreeOnJoinKey.check( + pkSide == 0 ? left : right, + pkSide == 0 ? leftColIdx : rightColIdx); + + double leftNDV = + isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(left, lBitSet, leftPred) : -1; + double rightNDV = + isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(right, rBitSet, rightPred) : -1; + + /* + * If the ndv of the PK - FK side don't match, and the PK side is a filter + * on the Key column then scale the NDV on the FK side. + * + * As described by Peter Boncz: http://databasearchitects.blogspot.com/ + * in such cases we can be off by a large margin in the Join cardinality + * estimate. The e.g. he provides is on the join of StoreSales and DateDim + * on the TPCDS dataset. Since the DateDim is populated for 20 years into + * the future, while the StoreSales only has 5 years worth of data, there + * are 40 times fewer distinct dates in StoreSales. + * + * In general it is hard to infer the range for the foreign key on an + * arbitrary expression. For e.g. the NDV for DayofWeek is the same + * irrespective of NDV on the number of unique days, whereas the + * NDV of Quarters has the same ratio as the NDV on the keys. + * + * But for expressions that apply only on columns that have the same NDV + * as the key (implying that they are alternate keys) we can apply the + * ratio. So in the case of StoreSales - DateDim joins for predicate on the + * d_date column we can apply the scaling factor. + */ + double ndvScalingFactor = 1.0; + if (isPKSideSimpleTree) { + ndvScalingFactor = pkSide == 0 ? leftNDV / rightNDV : rightNDV / leftNDV; + } if (pkSide == 0) { FKSideInfo fkInfo = new FKSideInfo(rightRowCount, @@ -325,9 +328,6 @@ private static boolean isKey(BitSet c, RelNode rel) { RexNode joinCond = joinFilters.get(0); - int leftColIdx; - int rightColIdx; - if (!(joinCond instanceof RexCall)) { return null; } @@ -339,7 +339,7 @@ private static boolean isKey(BitSet c, RelNode rel) { BitSet leftCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(0)); BitSet rightCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(1)); - if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1 ) { + if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1) { return null; } @@ -358,16 +358,16 @@ private static boolean isKey(BitSet c, RelNode rel) { rightCols = t; } - leftColIdx = leftCols.nextSetBit(0) - nSysFields; - rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft); + int leftColIdx = leftCols.nextSetBit(0) - nSysFields; + int rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft); return new Pair(leftColIdx, rightColIdx); } private static class IsSimpleTreeOnJoinKey extends RelVisitor { - int joinKey; - boolean simpleTree; + private int joinKey; + private boolean simpleTree; static boolean check(RelNode r, int joinKey) { IsSimpleTreeOnJoinKey v = new IsSimpleTreeOnJoinKey(joinKey); @@ -376,7 +376,6 @@ static boolean check(RelNode r, int joinKey) { } IsSimpleTreeOnJoinKey(int joinKey) { - super(); this.joinKey = joinKey; simpleTree = true; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java index 49d2ee5..dfbae35 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java @@ -23,8 +23,8 @@ import java.util.Map; import java.util.Set; +import com.google.common.collect.ImmutableMap; import net.hydromatic.optiq.BuiltinMethod; - import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil.JoinLeafPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil.JoinPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; @@ -36,18 +36,14 @@ import org.eigenbase.rel.metadata.RelMetadataProvider; import org.eigenbase.rel.metadata.RelMetadataQuery; import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexUtil; import org.eigenbase.util.Pair; -import com.google.common.collect.ImmutableMap; - public class HiveRelMdSelectivity extends RelMdSelectivity { public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( BuiltinMethod.SELECTIVITY.method, new HiveRelMdSelectivity()); protected HiveRelMdSelectivity() { - super(); } public Double getSelectivity(HiveTableScanRel t, RexNode predicate) { @@ -60,14 +56,13 @@ public Double getSelectivity(HiveTableScanRel t, RexNode predicate) { } public Double getSelectivity(HiveJoinRel j, RexNode predicate) { - if (j.getJoinType().equals(JoinRelType.INNER)) { + if (j.getJoinType() == JoinRelType.INNER) { return computeInnerJoinSelectivity(j, predicate); } return 1.0; } private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { - double ndvCrossProduct = 1; Pair predInfo = getCombinedPredicateForJoin(j, predicate); if (!predInfo.getKey()) { @@ -81,7 +76,6 @@ private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { combinedPredicate); ImmutableMap.Builder colStatMapBuilder = ImmutableMap .builder(); - ImmutableMap colStatMap; int rightOffSet = j.getLeft().getRowType().getFieldCount(); // 1. Update Col Stats Map with col stats for columns from left side of @@ -97,25 +91,28 @@ private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { colStatMapBuilder.put(rjk + rightOffSet, HiveRelMdDistinctRowCount.getDistinctRowCount(j.getRight(), rjk)); } - colStatMap = colStatMapBuilder.build(); + ImmutableMap colStatMap = colStatMapBuilder.build(); // 3. Walk through the Join Condition Building NDV for selectivity // NDV of the join can not exceed the cardinality of cross join. List peLst = jpi.getEquiJoinPredicateElements(); int noOfPE = peLst.size(); + double ndvCrossProduct = 1; if (noOfPE > 0) { ndvCrossProduct = exponentialBackoff(peLst, colStatMap); - if (j.isLeftSemiJoin()) - ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()), - ndvCrossProduct); - else - ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()) - * RelMetadataQuery.getRowCount(j.getRight()), ndvCrossProduct); + if (j.isLeftSemiJoin()) { + ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()), ndvCrossProduct); + } else { + ndvCrossProduct = Math.min( + RelMetadataQuery.getRowCount(j.getLeft()) * RelMetadataQuery.getRowCount(j.getRight()), + ndvCrossProduct); + + } } // 4. Join Selectivity = 1/NDV - return (1 / ndvCrossProduct); + return 1 / ndvCrossProduct; } // 3.2 if conjunctive predicate elements are more than one, then walk @@ -136,16 +133,16 @@ private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { // where pex is the predicate element of join condition with max ndv. // ndv(pe) = max(NDV(left.Expr), NDV(right.Expr)) // NDV(expr) = max(NDV( expr args)) - protected double logSmoothing(List peLst, ImmutableMap colStatMap) { + protected double logSmoothing(List peLst, + ImmutableMap colStatMap) { int noOfPE = peLst.size(); double ndvCrossProduct = getMaxNDVForJoinSelectivity(peLst.get(0), colStatMap); if (noOfPE > 1) { double maxNDVSoFar = ndvCrossProduct; - double ndvToBeSmoothed; - double tmpNDV; for (int i = 1; i < noOfPE; i++) { - tmpNDV = getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap); + double tmpNDV = getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap); + double ndvToBeSmoothed; if (tmpNDV > maxNDVSoFar) { ndvToBeSmoothed = maxNDVSoFar; maxNDVSoFar = tmpNDV; @@ -154,10 +151,11 @@ protected double logSmoothing(List peLst, ImmutableMap 3) + if (ndvToBeSmoothed > 3) { ndvCrossProduct *= Math.log(ndvToBeSmoothed); - else + } else { ndvCrossProduct *= ndvToBeSmoothed; + } } } return ndvCrossProduct; @@ -171,8 +169,8 @@ protected double exponentialBackoff(List peLst, ImmutableMap colStatMap) { int noOfPE = peLst.size(); List ndvs = new ArrayList(noOfPE); - for (int i = 0; i < noOfPE; i++) { - ndvs.add(getMaxNDVForJoinSelectivity(peLst.get(i), colStatMap)); + for (JoinLeafPredicateInfo aPeLst : peLst) { + ndvs.add(getMaxNDVForJoinSelectivity(aPeLst, colStatMap)); } Collections.sort(ndvs); Collections.reverse(ndvs); @@ -185,35 +183,30 @@ protected double exponentialBackoff(List peLst, } /** - * - * @param j - * @param additionalPredicate * @return if predicate is the join condition return (true, joinCond) * else return (false, minusPred) */ - private Pair getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPredicate) { - RexNode minusPred = RelMdUtil.minusPreds(j.getCluster().getRexBuilder(), additionalPredicate, - j.getCondition()); + private Pair getCombinedPredicateForJoin(HiveJoinRel j, + RexNode additionalPredicate) { + RexNode minusPred = + RelMdUtil.minusPreds(j.getCluster().getRexBuilder(), additionalPredicate, j.getCondition()); if (minusPred != null) { List minusList = new ArrayList(); minusList.add(j.getCondition()); minusList.add(minusPred); - return new Pair(false, minusPred); + return new Pair(false, minusPred); } - return new Pair(true,j.getCondition()); + return new Pair(true, j.getCondition()); } /** * Compute Max NDV to determine Join Selectivity. - * - * @param jlpi + * * @param colStatMap * Immutable Map of Projection Index (in Join Schema) to Column Stat - * @param rightProjOffSet - * @return */ private static Double getMaxNDVForJoinSelectivity(JoinLeafPredicateInfo jlpi, ImmutableMap colStatMap) { @@ -229,16 +222,16 @@ private static Double getMaxNDVForJoinSelectivity(JoinLeafPredicateInfo jlpi, private static Double getMaxNDVFromProjections(Map colStatMap, Set projectionSet, Double defaultMaxNDV) { - Double colNDV = null; Double maxNDVSoFar = defaultMaxNDV; for (Integer projIndx : projectionSet) { - colNDV = colStatMap.get(projIndx); - if (colNDV > maxNDVSoFar) + Double colNDV = colStatMap.get(projIndx); + if (colNDV > maxNDVSoFar) { maxNDVSoFar = colNDV; + } } return maxNDVSoFar; } - + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java index 06ff584..2d2616a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java @@ -51,7 +51,7 @@ /* * Infer Uniquenes if: - rowCount(col) = ndv(col) - TBD for numerics: max(col) * - min(col) = rowCount(col) - * + * * Why are we intercepting ProjectRelbase and not TableScan? Because if we * have a method for TableScan, it will not know which columns to check for. * Inferring Uniqueness for all columns is very expensive right now. The flip @@ -94,14 +94,14 @@ if (cStat.getCountDistint() >= numRows) { isKey = true; } - if ( !isKey && cStat.getRange() != null && - cStat.getRange().maxValue != null && - cStat.getRange().minValue != null) { - double r = cStat.getRange().maxValue.doubleValue() - - cStat.getRange().minValue.doubleValue() + 1; - isKey = (numRows == r); + if (!isKey && cStat.getRange() != null + && cStat.getRange().maxValue != null + && cStat.getRange().minValue != null) { + double r = + cStat.getRange().maxValue.doubleValue() - cStat.getRange().minValue.doubleValue() + 1; + isKey = numRows == r; } - if ( isKey ) { + if (isKey) { BitSet key = new BitSet(); key.set(posMap.get(colStatsPos)); keys.add(key); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java index 98723a3..7cc5342 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java @@ -22,7 +22,6 @@ import java.util.Calendar; import net.hydromatic.avatica.ByteString; - import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; @@ -35,6 +34,8 @@ class ASTBuilder { + private ASTNode curr; + static ASTBuilder construct(int tokenType, String text) { ASTBuilder b = new ASTBuilder(); b.curr = createAST(tokenType, text); @@ -46,17 +47,14 @@ static ASTNode createAST(int tokenType, String text) { } static ASTNode destNode() { - return ASTBuilder - .construct(HiveParser.TOK_DESTINATION, "TOK_DESTINATION") - .add( - ASTBuilder.construct(HiveParser.TOK_DIR, "TOK_DIR").add(HiveParser.TOK_TMP_FILE, - "TOK_TMP_FILE")).node(); + return construct(HiveParser.TOK_DESTINATION, "TOK_DESTINATION").add( + construct(HiveParser.TOK_DIR, "TOK_DIR").add(HiveParser.TOK_TMP_FILE, "TOK_TMP_FILE")).node(); } static ASTNode table(TableAccessRelBase scan) { RelOptHiveTable hTbl = (RelOptHiveTable) scan.getTable(); - ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add( - ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") + ASTBuilder b = construct(HiveParser.TOK_TABREF, "TOK_TABREF").add( + construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()) .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName())); @@ -77,19 +75,19 @@ static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode c switch (joinType) { case INNER: if (semiJoin) { - b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); + b = construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); } else { - b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN"); + b = construct(HiveParser.TOK_JOIN, "TOK_JOIN"); } break; case LEFT: - b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN"); + b = construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN"); break; case RIGHT: - b = ASTBuilder.construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN"); + b = construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN"); break; case FULL: - b = ASTBuilder.construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN"); + b = construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN"); break; } @@ -98,40 +96,39 @@ static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode c } static ASTNode subQuery(ASTNode qry, String alias) { - return ASTBuilder.construct(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY").add(qry) + return construct(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY").add(qry) .add(HiveParser.Identifier, alias).node(); } static ASTNode qualifiedName(String tableName, String colName) { - ASTBuilder b = ASTBuilder - .construct(HiveParser.DOT, ".") + ASTBuilder b = construct(HiveParser.DOT, ".") .add( - ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( + construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( HiveParser.Identifier, tableName)).add(HiveParser.Identifier, colName); return b.node(); } static ASTNode unqualifiedName(String colName) { - ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( + ASTBuilder b = construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add( HiveParser.Identifier, colName); return b.node(); } static ASTNode where(ASTNode cond) { - return ASTBuilder.construct(HiveParser.TOK_WHERE, "TOK_WHERE").add(cond).node(); + return construct(HiveParser.TOK_WHERE, "TOK_WHERE").add(cond).node(); } static ASTNode having(ASTNode cond) { - return ASTBuilder.construct(HiveParser.TOK_HAVING, "TOK_HAVING").add(cond).node(); + return construct(HiveParser.TOK_HAVING, "TOK_HAVING").add(cond).node(); } static ASTNode limit(Object value) { - return ASTBuilder.construct(HiveParser.TOK_LIMIT, "TOK_LIMIT") + return construct(HiveParser.TOK_LIMIT, "TOK_LIMIT") .add(HiveParser.Number, value.toString()).node(); } static ASTNode selectExpr(ASTNode expr, String alias) { - return ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr) + return construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr) .add(HiveParser.Identifier, alias).node(); } @@ -181,7 +178,7 @@ static ASTNode literal(RexLiteral literal) { break; case BOOLEAN: val = literal.getValue3(); - type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE : HiveParser.KW_FALSE; + type = (Boolean) val ? HiveParser.KW_TRUE : HiveParser.KW_FALSE; break; case DATE: { val = literal.getValue(); @@ -192,13 +189,12 @@ static ASTNode literal(RexLiteral literal) { } break; case TIME: - case TIMESTAMP: { + case TIMESTAMP: val = literal.getValue(); type = HiveParser.TOK_TIMESTAMP; DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); val = df.format(((Calendar) val).getTime()); val = "'" + val + "'"; - } break; case NULL: type = HiveParser.TOK_NULL; @@ -211,8 +207,6 @@ static ASTNode literal(RexLiteral literal) { return (ASTNode) ParseDriver.adaptor.create(type, String.valueOf(val)); } - ASTNode curr; - ASTNode node() { return curr; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java index f5a704f..8e15400 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java @@ -64,24 +64,24 @@ public class ASTConverter { - private RelNode root; - private HiveAST hiveAST; - private RelNode from; - private FilterRelBase where; + private final RelNode root; + private final HiveAST hiveAST; + private RelNode from; + private FilterRelBase where; private AggregateRelBase groupBy; - private FilterRelBase having; - private ProjectRelBase select; - private SortRel order; - private SortRel limit; + private FilterRelBase having; + private ProjectRelBase select; + private SortRel order; + private SortRel limit; - private Schema schema; + private Schema schema; - private long derivedTableCount; + private long derivedTableCount; ASTConverter(RelNode root, long dtCounterInitVal) { this.root = root; hiveAST = new HiveAST(); - this.derivedTableCount = dtCounterInitVal; + derivedTableCount = dtCounterInitVal; } public static ASTNode convert(final RelNode relNode, List resultSchema) @@ -123,8 +123,9 @@ private ASTNode convert() { b.add(iRef.accept(new RexVisitor(schema))); } - if (!groupBy.getGroupSet().isEmpty()) + if (!groupBy.getGroupSet().isEmpty()) { hiveAST.groupBy = b.node(); + } schema = new Schema(schema, groupBy); } @@ -142,7 +143,7 @@ private ASTNode convert() { ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT"); if (select.getChildExps().isEmpty()) { - RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(new BigDecimal(1)); + RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.ONE); ASTNode selectExpr = ASTBuilder.selectExpr(ASTBuilder.literal(r), "1"); b.add(selectExpr); } else { @@ -172,8 +173,7 @@ private ASTNode convert() { private void convertLimitToASTNode(HiveSortRel limit) { if (limit != null) { - HiveSortRel hiveLimit = (HiveSortRel) limit; - RexNode limitExpr = hiveLimit.getFetchExpr(); + RexNode limitExpr = limit.getFetchExpr(); if (limitExpr != null) { Object val = ((RexLiteral) limitExpr).getValue2(); hiveAST.limit = ASTBuilder.limit(val); @@ -182,46 +182,44 @@ private void convertLimitToASTNode(HiveSortRel limit) { } private void convertOBToASTNode(HiveSortRel order) { - if (order != null) { - HiveSortRel hiveSort = (HiveSortRel) order; - if (!hiveSort.getCollation().getFieldCollations().isEmpty()) { - // 1 Add order by token - ASTNode orderAst = ASTBuilder.createAST(HiveParser.TOK_ORDERBY, "TOK_ORDERBY"); - - schema = new Schema((HiveSortRel) hiveSort); - Map obRefToCallMap = hiveSort.getInputRefToCallMap(); - RexNode obExpr; - ASTNode astCol; - for (RelFieldCollation c : hiveSort.getCollation().getFieldCollations()) { - - // 2 Add Direction token - ASTNode directionAST = c.getDirection() == RelFieldCollation.Direction.ASCENDING ? ASTBuilder - .createAST(HiveParser.TOK_TABSORTCOLNAMEASC, "TOK_TABSORTCOLNAMEASC") : ASTBuilder - .createAST(HiveParser.TOK_TABSORTCOLNAMEDESC, "TOK_TABSORTCOLNAMEDESC"); - - // 3 Convert OB expr (OB Expr is usually an input ref except for top - // level OB; top level OB will have RexCall kept in a map.) - obExpr = null; - if (obRefToCallMap != null) - obExpr = obRefToCallMap.get(c.getFieldIndex()); + if (order != null && !order.getCollation().getFieldCollations().isEmpty()) { + // 1 Add order by token + ASTNode orderAst = ASTBuilder.createAST(HiveParser.TOK_ORDERBY, "TOK_ORDERBY"); + + schema = new Schema(order); + Map obRefToCallMap = order.getInputRefToCallMap(); + for (RelFieldCollation c : order.getCollation().getFieldCollations()) { + + // 2 Add Direction token + ASTNode directionAST = + c.getDirection() == RelFieldCollation.Direction.ASCENDING + ? ASTBuilder.createAST(HiveParser.TOK_TABSORTCOLNAMEASC, "TOK_TABSORTCOLNAMEASC") + : ASTBuilder.createAST(HiveParser.TOK_TABSORTCOLNAMEDESC, "TOK_TABSORTCOLNAMEDESC"); + + // 3 Convert OB expr (OB Expr is usually an input ref except for top + // level OB; top level OB will have RexCall kept in a map.) + RexNode obExpr = null; + if (obRefToCallMap != null) { + obExpr = obRefToCallMap.get(c.getFieldIndex()); + } - if (obExpr != null) { - astCol = obExpr.accept(new RexVisitor(schema)); - } else { - ColumnInfo cI = schema.get(c.getFieldIndex()); + ASTNode astCol; + if (obExpr == null) { + ColumnInfo cI = schema.get(c.getFieldIndex()); /* * The RowResolver setup for Select drops Table associations. So * setup ASTNode on unqualified name. */ - astCol = ASTBuilder.unqualifiedName(cI.column); - } - - // 4 buildup the ob expr AST - directionAST.addChild(astCol); - orderAst.addChild(directionAST); + astCol = ASTBuilder.unqualifiedName(cI.column); + } else { + astCol = obExpr.accept(new RexVisitor(schema)); } - hiveAST.order = orderAst; + + // 4 buildup the ob expr AST + directionAST.addChild(astCol); + orderAst.addChild(directionAST); } + hiveAST.order = orderAst; } } @@ -245,14 +243,15 @@ private QueryBlockInfo convertSource(RelNode r) { ASTNode cond = join.getCondition().accept(new RexVisitor(s)); boolean semiJoin = join instanceof SemiJoinRel; ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin); - if (semiJoin) + if (semiJoin) { s = left.schema; + } } else if (r instanceof UnionRelBase) { - RelNode leftInput = ((UnionRelBase) r).getInput(0); - RelNode rightInput = ((UnionRelBase) r).getInput(1); + RelNode leftInput = r.getInput(0); + RelNode rightInput = r.getInput(1); - ASTConverter leftConv = new ASTConverter(leftInput, this.derivedTableCount); - ASTConverter rightConv = new ASTConverter(rightInput, this.derivedTableCount); + ASTConverter leftConv = new ASTConverter(leftInput, derivedTableCount); + ASTConverter rightConv = new ASTConverter(rightInput, derivedTableCount); ASTNode leftAST = leftConv.convert(); ASTNode rightAST = rightConv.convert(); @@ -262,7 +261,7 @@ private QueryBlockInfo convertSource(RelNode r) { ast = ASTBuilder.subQuery(unionAST, sqAlias); s = new Schema((UnionRelBase) r, sqAlias); } else { - ASTConverter src = new ASTConverter(r, this.derivedTableCount); + ASTConverter src = new ASTConverter(r, derivedTableCount); ASTNode srcAST = src.convert(); String sqAlias = nextAlias(); s = src.getRowSchema(sqAlias); @@ -275,18 +274,19 @@ private QueryBlockInfo convertSource(RelNode r) { public void handle(FilterRelBase filter) { RelNode child = filter.getChild(); - if (child instanceof AggregateRelBase && !((AggregateRelBase) child).getGroupSet().isEmpty()) { - ASTConverter.this.having = filter; + if (child instanceof AggregateRelBase && !((AggregateRelBase) child).getGroupSet() + .isEmpty()) { + having = filter; } else { - ASTConverter.this.where = filter; + where = filter; } } public void handle(ProjectRelBase project) { - if (ASTConverter.this.select == null) { - ASTConverter.this.select = project; + if (select == null) { + select = project; } else { - ASTConverter.this.from = project; + from = project; } } @@ -294,32 +294,33 @@ public void handle(ProjectRelBase project) { public void visit(RelNode node, int ordinal, RelNode parent) { if (node instanceof TableAccessRelBase) { - ASTConverter.this.from = node; + from = node; } else if (node instanceof FilterRelBase) { handle((FilterRelBase) node); } else if (node instanceof ProjectRelBase) { handle((ProjectRelBase) node); } else if (node instanceof JoinRelBase) { - ASTConverter.this.from = node; + from = node; } else if (node instanceof UnionRelBase) { - ASTConverter.this.from = node; + from = node; } else if (node instanceof AggregateRelBase) { - ASTConverter.this.groupBy = (AggregateRelBase) node; + groupBy = (AggregateRelBase) node; } else if (node instanceof SortRel) { - if (ASTConverter.this.select != null) { - ASTConverter.this.from = node; - } else { + if (select == null) { SortRel hiveSortRel = (SortRel) node; - if (hiveSortRel.getCollation().getFieldCollations().isEmpty()) - ASTConverter.this.limit = hiveSortRel; - else - ASTConverter.this.order = hiveSortRel; + if (hiveSortRel.getCollation().getFieldCollations().isEmpty()) { + limit = hiveSortRel; + } else { + order = hiveSortRel; + } + } else { + from = node; } } /* * once the source node is reached; stop traversal for this QB */ - if (ASTConverter.this.from == null) { + if (from == null) { node.childrenAccept(this); } } @@ -348,10 +349,11 @@ public ASTNode visitInputRef(RexInputRef inputRef) { return (ASTNode) ParseDriver.adaptor.dupTree(cI.agg); } - if (cI.table == null || cI.table.isEmpty()) + if (cI.table == null || cI.table.isEmpty()) { return ASTBuilder.unqualifiedName(cI.column); - else + } else { return ASTBuilder.qualifiedName(cI.table, cI.column); + } } @@ -361,7 +363,6 @@ public ASTNode visitLiteral(RexLiteral literal) { } private ASTNode getPSpecAST(RexWindow window) { - ASTNode pSpecAst = null; ASTNode dByAst = null; if (window.partitionKeys != null && !window.partitionKeys.isEmpty()) { @@ -385,27 +386,31 @@ private ASTNode getPSpecAST(RexWindow window) { } } + ASTNode pSpecAst = null; if (dByAst != null || oByAst != null) { pSpecAst = ASTBuilder.createAST(HiveParser.TOK_PARTITIONINGSPEC, "TOK_PARTITIONINGSPEC"); - if (dByAst != null) + if (dByAst != null) { pSpecAst.addChild(dByAst); - if (oByAst != null) + } + if (oByAst != null) { pSpecAst.addChild(oByAst); + } } return pSpecAst; } private ASTNode getWindowBound(RexWindowBound wb) { - ASTNode wbAST = null; + ASTNode wbAST; if (wb.isCurrentRow()) { wbAST = ASTBuilder.createAST(HiveParser.KW_CURRENT, "CURRENT"); } else { - if (wb.isPreceding()) + if (wb.isPreceding()) { wbAST = ASTBuilder.createAST(HiveParser.KW_PRECEDING, "PRECEDING"); - else + } else { wbAST = ASTBuilder.createAST(HiveParser.KW_FOLLOWING, "FOLLOWING"); + } if (wb.isUnbounded()) { wbAST.addChild(ASTBuilder.createAST(HiveParser.KW_UNBOUNDED, "UNBOUNDED")); } else { @@ -418,7 +423,6 @@ private ASTNode getWindowBound(RexWindowBound wb) { } private ASTNode getWindowRangeAST(RexWindow window) { - ASTNode wRangeAst = null; ASTNode startAST = null; RexWindowBound ub = window.getUpperBound(); @@ -432,16 +436,20 @@ private ASTNode getWindowRangeAST(RexWindow window) { endAST = getWindowBound(lb); } + ASTNode wRangeAst = null; if (startAST != null || endAST != null) { // NOTE: in Hive AST Rows->Range(Physical) & Range -> Values (logical) - if (window.isRows()) + if (window.isRows()) { wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWRANGE, "TOK_WINDOWRANGE"); - else + } else { wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWVALUES, "TOK_WINDOWVALUES"); - if (startAST != null) + } + if (startAST != null) { wRangeAst.addChild(startAST); - if (endAST != null) + } + if (endAST != null) { wRangeAst.addChild(endAST); + } } return wRangeAst; @@ -464,10 +472,12 @@ public ASTNode visitOver(RexOver over) { final RexWindow window = over.getWindow(); final ASTNode wPSpecAst = getPSpecAST(window); final ASTNode wRangeAst = getWindowRangeAST(window); - if (wPSpecAst != null) + if (wPSpecAst != null) { wSpec.addChild(wPSpecAst); - if (wRangeAst != null) + } + if (wRangeAst != null) { wSpec.addChild(wRangeAst); + } return wUDAFAst; } @@ -484,8 +494,9 @@ public ASTNode visitCall(RexCall call) { HiveToken ht = TypeConverter.hiveToken(call.getType()); ASTBuilder astBldr = ASTBuilder.construct(ht.type, ht.text); if (ht.args != null) { - for (String castArg : ht.args) + for (String castArg : ht.args) { astBldr.add(HiveParser.Identifier, castArg); + } } astNodeLst.add(astBldr.node()); } @@ -494,10 +505,11 @@ public ASTNode visitCall(RexCall call) { astNodeLst.add(operand.accept(this)); } - if (isFlat(call)) + if (isFlat(call)) { return SqlFunctionConverter.buildAST(op, astNodeLst, 0); - else + } else { return SqlFunctionConverter.buildAST(op, astNodeLst); + } } } @@ -505,8 +517,7 @@ public ASTNode visitCall(RexCall call) { Schema schema; ASTNode ast; - public QueryBlockInfo(Schema schema, ASTNode ast) { - super(); + QueryBlockInfo(Schema schema, ASTNode ast) { this.schema = schema; this.ast = ast; } @@ -538,7 +549,6 @@ public QueryBlockInfo(Schema schema, ASTNode ast) { } } - @SuppressWarnings("unchecked") Schema(Schema left, Schema right) { for (ColumnInfo cI : Iterables.concat(left, right)) { add(cI); @@ -571,12 +581,11 @@ public QueryBlockInfo(Schema schema, ASTNode ast) { * 1. ProjectRel will always be child of SortRel.
* 2. In Optiq every projection in ProjectRelBase is uniquely named * (unambigous) without using table qualifier (table name).
- * + * * @param order * Hive Sort Rel Node - * @return Schema */ - public Schema(HiveSortRel order) { + Schema(HiveSortRel order) { ProjectRelBase select = (ProjectRelBase) order.getChild(); for (String projName : select.getRowType().getFieldNames()) { add(new ColumnInfo(null, projName)); @@ -593,21 +602,19 @@ public Schema(HiveSortRel order) { ASTNode agg; ColumnInfo(String table, String column) { - super(); this.table = table; this.column = column; } ColumnInfo(String table, ASTNode agg) { - super(); this.table = table; this.agg = agg; } ColumnInfo(String alias, ColumnInfo srcCol) { - this.table = alias; - this.column = srcCol.column; - this.agg = srcCol.agg; + table = alias; + column = srcCol.column; + agg = srcCol.agg; } } @@ -640,10 +647,8 @@ public ASTNode getAST() { public ASTNode getUnionAllAST(ASTNode leftAST, ASTNode rightAST) { - ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST) + return ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST) .add(rightAST).node(); - - return unionTokAST; } public static boolean isFlat(RexCall call) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java index e6e491f..7574df8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java @@ -43,9 +43,9 @@ */ public class ExprNodeConverter extends RexVisitorImpl { - RelDataType rType; - String tabAlias; - boolean partitioningExpr; + private final RelDataType rType; + private final String tabAlias; + private final boolean partitioningExpr; public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitioningExpr) { super(true); @@ -56,7 +56,7 @@ public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitionin * on whether a column is a partition column or not, whether a column is a * virtual column or not. */ - assert partitioningExpr == true; + assert partitioningExpr; this.tabAlias = tabAlias; this.rType = rType; this.partitioningExpr = partitioningExpr; @@ -71,7 +71,6 @@ public ExprNodeDesc visitInputRef(RexInputRef inputRef) { @Override public ExprNodeDesc visitCall(RexCall call) { - ExprNodeGenericFuncDesc gfDesc = null; if (!deep) { return null; @@ -85,8 +84,9 @@ public ExprNodeDesc visitCall(RexCall call) { // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the // exprnode + ExprNodeGenericFuncDesc gfDesc; if (ASTConverter.isFlat(call)) { - ArrayList tmpExprArgs = new ArrayList(); + List tmpExprArgs = new ArrayList(); tmpExprArgs.addAll(args.subList(0, 2)); gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()), SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType()), tmpExprArgs); @@ -111,29 +111,29 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) { switch (literal.getType().getSqlTypeName()) { case BOOLEAN: - return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.valueOf(RexLiteral - .booleanValue(literal))); + return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, + RexLiteral.booleanValue(literal)); case TINYINT: - return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, Byte.valueOf(((Number) literal - .getValue3()).byteValue())); + return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, + ((Number) literal.getValue3()).byteValue()); case SMALLINT: return new ExprNodeConstantDesc(TypeInfoFactory.shortTypeInfo, - Short.valueOf(((Number) literal.getValue3()).shortValue())); + ((Number) literal.getValue3()).shortValue()); case INTEGER: return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, - Integer.valueOf(((Number) literal.getValue3()).intValue())); + ((Number) literal.getValue3()).intValue()); case BIGINT: - return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, Long.valueOf(((Number) literal - .getValue3()).longValue())); + return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, + ((Number) literal.getValue3()).longValue()); case FLOAT: return new ExprNodeConstantDesc(TypeInfoFactory.floatTypeInfo, - Float.valueOf(((Number) literal.getValue3()).floatValue())); + ((Number) literal.getValue3()).floatValue()); case DOUBLE: return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo, - Double.valueOf(((Number) literal.getValue3()).doubleValue())); + ((Number) literal.getValue3()).doubleValue()); case DATE: return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, - new Date(((Calendar)literal.getValue()).getTimeInMillis())); + new Date(((Calendar) literal.getValue()).getTimeInMillis())); case TIMESTAMP: return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, literal.getValue3()); case BINARY: diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java index 406c18e..1396a33 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinCondTypeCheckProcFactory.java @@ -48,7 +48,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; /** - * JoinCondTypeCheckProcFactory is used by Optiq planner(CBO) to generate Join Conditions from Join Condition AST. + * JoinCondTypeCheckProcFactory is used by Optiq planner(CBO) to generate Join Conditions from + * Join Condition AST. * Reasons for sub class: * 1. Additional restrictions on what is supported in Join Conditions * 2. Column handling is different @@ -90,14 +91,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } - assert (expr.getChildCount() == 1); + assert expr.getChildCount() == 1; String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); - boolean qualifiedAccess = (parent != null && parent.getType() == HiveParser.DOT); + boolean qualifiedAccess = parent != null && parent.getType() == HiveParser.DOT; - ColumnInfo colInfo = null; if (!qualifiedAccess) { - colInfo = getColInfo(ctx, null, tableOrCol, expr); + ColumnInfo colInfo = getColInfo(ctx, null, tableOrCol, expr); // It's a column. return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol()); @@ -112,24 +112,24 @@ private static boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNo throws SemanticException { int tblAliasCnt = 0; for (RowResolver rr : ctx.getInputRRList()) { - if (rr.hasTableAlias(tabName)) + if (rr.hasTableAlias(tabName)) { tblAliasCnt++; + } } if (tblAliasCnt > 1) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); } - return (tblAliasCnt == 1) ? true : false; + return tblAliasCnt == 1; } private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { - ColumnInfo tmp; ColumnInfo cInfoToRet = null; for (RowResolver rr : ctx.getInputRRList()) { - tmp = rr.get(tabName, colAlias); + ColumnInfo tmp = rr.get(tabName, colAlias); if (tmp != null) { if (cInfoToRet != null) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); @@ -144,7 +144,7 @@ private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, Strin /** * Factory method to get ColumnExprProcessor. - * + * * @return ColumnExprProcessor. */ @Override @@ -187,11 +187,10 @@ protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode ex private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { - ColumnInfo tmp; ColumnInfo cInfoToRet = null; for (RowResolver rr : ctx.getInputRRList()) { - tmp = rr.get(tabName, colAlias); + ColumnInfo tmp = rr.get(tabName, colAlias); if (tmp != null) { if (cInfoToRet != null) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); @@ -222,23 +221,19 @@ protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, F if (genericUDFargsRefersToBothInput(genericUDF, children, jCtx.getInputRRList())) { throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); } - } else if (genericUDF instanceof GenericUDFBaseCompare) { - // Comparisons of non literals LHS/RHS can not refer to inputs from - // both sides - if (children.size() == 2 && !(children.get(0) instanceof ExprNodeConstantDesc) - && !(children.get(1) instanceof ExprNodeConstantDesc)) { - if (comparisonUDFargsRefersToBothInput((GenericUDFBaseCompare) genericUDF, children, - jCtx.getInputRRList())) { - throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); - } - } + } else if (genericUDF instanceof GenericUDFBaseCompare + && children.size() == 2 + && !(children.get(0) instanceof ExprNodeConstantDesc) + && !(children.get(1) instanceof ExprNodeConstantDesc) + && comparisonUDFargsRefersToBothInput((GenericUDFBaseCompare) genericUDF, + children, jCtx.getInputRRList())) { + throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr)); } } } private static boolean genericUDFargsRefersToBothInput(GenericUDF udf, List children, List inputRRList) { - boolean argsRefersToBothInput = false; Map hasCodeToColDescMap = new HashMap(); for (ExprNodeDesc child : children) { @@ -246,15 +241,16 @@ private static boolean genericUDFargsRefersToBothInput(GenericUDF udf, } Set inputRef = getInputRef(hasCodeToColDescMap.values(), inputRRList); - if (inputRef.size() > 1) + boolean argsRefersToBothInput = false; + if (inputRef.size() > 1) { argsRefersToBothInput = true; + } return argsRefersToBothInput; } private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare comparisonUDF, List children, List inputRRList) { - boolean argsRefersToBothInput = false; Map lhsHashCodeToColDescMap = new HashMap(); Map rhsHashCodeToColDescMap = new HashMap(); @@ -263,37 +259,36 @@ private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare Set lhsInputRef = getInputRef(lhsHashCodeToColDescMap.values(), inputRRList); Set rhsInputRef = getInputRef(rhsHashCodeToColDescMap.values(), inputRRList); - if (lhsInputRef.size() > 1 || rhsInputRef.size() > 1) + boolean argsRefersToBothInput = false; + if (lhsInputRef.size() > 1 || rhsInputRef.size() > 1) { argsRefersToBothInput = true; + } return argsRefersToBothInput; } private static Set getInputRef(Collection colDescSet, List inputRRList) { - String tableAlias; - RowResolver inputRR; Set inputLineage = new HashSet(); for (ExprNodeDesc col : colDescSet) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) col; - tableAlias = colDesc.getTabAlias(); + String tableAlias = colDesc.getTabAlias(); for (int i = 0; i < inputRRList.size(); i++) { - inputRR = inputRRList.get(i); + RowResolver inputRR = inputRRList.get(i); // If table Alias is present check if InputRR has that table and then // check for internal name // else if table alias is null then check with internal name in all // inputRR. - if (tableAlias != null) { - if (inputRR.hasTableAlias(tableAlias)) { - if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { - inputLineage.add(i); - } + if (tableAlias == null) { + if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { + inputLineage.add(i); } } else { - if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { + if (inputRR.hasTableAlias(tableAlias) + && inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) { inputLineage.add(i); } } @@ -306,7 +301,7 @@ private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare /** * Factory method to get DefaultExprProcessor. - * + * * @return DefaultExprProcessor. */ @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java index fdee66b..df3e83f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/JoinTypeCheckCtx.java @@ -27,7 +27,8 @@ import com.google.common.collect.ImmutableList; /** - * JoinTypeCheckCtx is used by Optiq planner(CBO) to generate Join Conditions from Join Condition AST. + * JoinTypeCheckCtx is used by Optiq planner(CBO) to generate Join Conditions from + * Join Condition AST. * Reasons for sub class: * 1. Join Conditions can not handle: * a. Stateful Functions @@ -55,8 +56,8 @@ public JoinTypeCheckCtx(RowResolver leftRR, RowResolver rightRR, JoinType hiveJo throws SemanticException { super(RowResolver.getCombinedRR(leftRR, rightRR), false, false, false, false, false, false, false, false); - this.inputRRLst = ImmutableList.of(leftRR, rightRR); - this.outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) + inputRRLst = ImmutableList.of(leftRR, rightRR); + outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) || (hiveJoinType == JoinType.FULLOUTER); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java index 3d90ae7..1602b2f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java @@ -86,8 +86,9 @@ private static void convertOpTree(RelNode rel, RelNode parent) { throw new RuntimeException("Found RelSubset"); } else if (rel instanceof SetOpRel) { // TODO: Handle more than 2 inputs for setop - if (!validSetopParent(rel, parent)) + if (!validSetopParent(rel, parent)) { introduceDerivedTable(rel, parent); + } SetOpRel setopRel = (SetOpRel) rel; for (RelNode inputRel : setopRel.getInputs()) { @@ -107,10 +108,8 @@ private static void convertOpTree(RelNode rel, RelNode parent) { if (!validSortChild((HiveSortRel) rel)) { introduceDerivedTable(((HiveSortRel) rel).getChild(), rel); } - } else if (rel instanceof HiveAggregateRel) { - if (!validGBParent(rel, parent)) { - introduceDerivedTable(rel, parent); - } + } else if (rel instanceof HiveAggregateRel && !validGBParent(rel, parent)) { + introduceDerivedTable(rel, parent); } } @@ -178,11 +177,11 @@ private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, } List newSelAliases = new ArrayList(); - String colAlias; for (int i = 0; i < rootChildExps.size(); i++) { - colAlias = resultSchema.get(i).getName(); - if (colAlias.startsWith("_")) + String colAlias = resultSchema.get(i).getName(); + if (colAlias.startsWith("_")) { colAlias = colAlias.substring(1); + } newSelAliases.add(colAlias); } @@ -200,10 +199,8 @@ private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, private static RelNode introduceDerivedTable(final RelNode rel) { List projectList = HiveOptiqUtil.getProjsFromBelowAsInputRef(rel); - HiveProjectRel select = HiveProjectRel.create(rel.getCluster(), rel, projectList, + return HiveProjectRel.create(rel.getCluster(), rel, projectList, rel.getRowType(), rel.getCollationList()); - - return select; } private static void introduceDerivedTable(final RelNode rel, RelNode parent) { @@ -245,7 +242,7 @@ private static boolean validJoinParent(RelNode joinNode, RelNode parent) { private static boolean validFilterParent(RelNode filterNode, RelNode parent) { boolean validParent = true; - // TOODO: Verify GB having is not a seperate filter (if so we shouldn't + // TODO: Verify GB having is not a seperate filter (if so we shouldn't // introduce derived table) if (parent instanceof FilterRelBase || parent instanceof JoinRelBase || parent instanceof SetOpRel) { @@ -258,11 +255,13 @@ private static boolean validFilterParent(RelNode filterNode, RelNode parent) { private static boolean validGBParent(RelNode gbNode, RelNode parent) { boolean validParent = true; - // TOODO: Verify GB having is not a seperate filter (if so we shouldn't + // TODO: Verify GB having is not a seperate filter (if so we shouldn't // introduce derived table) - if (parent instanceof JoinRelBase || parent instanceof SetOpRel + if (parent instanceof JoinRelBase + || parent instanceof SetOpRel || parent instanceof AggregateRelBase - || (parent instanceof FilterRelBase && ((AggregateRelBase) gbNode).getGroupSet().isEmpty())) { + || (parent instanceof FilterRelBase && ((AggregateRelBase) gbNode).getGroupSet() + .isEmpty())) { validParent = false; } @@ -273,8 +272,9 @@ private static boolean validSortParent(RelNode sortNode, RelNode parent) { boolean validParent = true; if (parent != null && !(parent instanceof ProjectRelBase) - && !((parent instanceof SortRel) || HiveOptiqUtil.orderRelNode(parent))) + && !((parent instanceof SortRel) || HiveOptiqUtil.orderRelNode(parent))) { validParent = false; + } return validParent; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java index ec85603..8d96113 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java @@ -23,15 +23,12 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import net.hydromatic.avatica.ByteString; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -79,38 +76,37 @@ import com.google.common.collect.ImmutableMap; public class RexNodeConverter { - private static final Log LOG = LogFactory.getLog(RexNodeConverter.class); private static class InputCtx { - private final RelDataType m_optiqInpDataType; - private final ImmutableMap m_hiveNameToPosMap; - private final RowResolver m_hiveRR; - private final int m_offsetInOptiqSchema; + private final RelDataType optiqInpDataType; + private final ImmutableMap hiveNameToPosMap; + private final RowResolver hiveRR; + private final int offsetInOptiqSchema; private InputCtx(RelDataType optiqInpDataType, ImmutableMap hiveNameToPosMap, RowResolver hiveRR, int offsetInOptiqSchema) { - m_optiqInpDataType = optiqInpDataType; - m_hiveNameToPosMap = hiveNameToPosMap; - m_hiveRR = hiveRR; - m_offsetInOptiqSchema = offsetInOptiqSchema; + this.optiqInpDataType = optiqInpDataType; + this.hiveNameToPosMap = hiveNameToPosMap; + this.hiveRR = hiveRR; + this.offsetInOptiqSchema = offsetInOptiqSchema; } - }; + } - private final RelOptCluster m_cluster; - private final ImmutableList m_inputCtxs; - private final boolean m_flattenExpr; + private final RelOptCluster cluster; + private final ImmutableList inputCtxs; + private final boolean flattenExpr; public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap nameToPosMap, int offset, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); + this.flattenExpr = flattenExpr; } public RexNodeConverter(RelOptCluster cluster, List inpCtxLst, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + inputCtxs = ImmutableList.builder().addAll(inpCtxLst).build(); + this.flattenExpr = flattenExpr; } public RexNode convert(ExprNodeDesc expr) throws SemanticException { @@ -134,7 +130,7 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept RexNode rexNode = convert(fieldDesc.getDesc()); if (rexNode instanceof RexCall) { // regular case of accessing nested field in a column - return m_cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); + return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); } else { // This may happen for schema-less tables, where columns are dynamically // supplied by serdes. @@ -144,17 +140,15 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept } private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticException { - ExprNodeDesc tmpExprNode; - RexNode tmpRN; List childRexNodeLst = new LinkedList(); - Builder argTypeBldr = ImmutableList. builder(); + Builder argTypeBldr = ImmutableList.builder(); // TODO: 1) Expand to other functions as needed 2) What about types other than primitive. TypeInfo tgtDT = null; GenericUDF tgtUdf = func.getGenericUDF(); - boolean isNumeric = tgtUdf instanceof GenericUDFBaseNumeric, - isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare; + boolean isNumeric = tgtUdf instanceof GenericUDFBaseNumeric; + boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare; if (isNumeric) { tgtDT = func.getTypeInfo(); @@ -167,7 +161,7 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep for (ExprNodeDesc childExpr : func.getChildren()) { - tmpExprNode = childExpr; + ExprNodeDesc tmpExprNode = childExpr; if (tgtDT != null && TypeInfoUtils.isConversionRequiredForComparison(tgtDT, childExpr.getTypeInfo())) { if (isCompare) { @@ -176,7 +170,7 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep } else if (isNumeric) { // For numeric, we'll do minimum necessary cast - if we cast to the type // of expression, bad things will happen. - GenericUDFBaseNumeric numericUdf = (GenericUDFBaseNumeric)tgtUdf; + GenericUDFBaseNumeric numericUdf = (GenericUDFBaseNumeric) tgtUdf; PrimitiveTypeInfo minArgType = numericUdf.deriveMinArgumentCast(childExpr, tgtDT); tmpExprNode = ParseUtils.createConversionCast(childExpr, minArgType); } else { @@ -184,32 +178,31 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep } } - argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), m_cluster.getTypeFactory())); - tmpRN = convert(tmpExprNode); + argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory())); + RexNode tmpRN = convert(tmpExprNode); childRexNodeLst.add(tmpRN); } // See if this is an explicit cast. - RexNode expr = null; - RelDataType retType = null; - expr = handleExplicitCast(func, childRexNodeLst); + RexNode expr = handleExplicitCast(func, childRexNodeLst); + RelDataType retType; if (expr == null) { // This is not a cast; process the function. - retType = TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()); + retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()); SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType); - expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); + expr = cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); } else { retType = expr.getType(); } // TODO: Cast Function in Optiq have a bug where it infertype on cast throws // an exception - if (m_flattenExpr && (expr instanceof RexCall) + if (flattenExpr && (expr instanceof RexCall) && !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) { RexCall call = (RexCall) expr; - expr = m_cluster.getRexBuilder().makeCall(retType, call.getOperator(), + expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(), RexUtil.flatten(call.getOperands(), call.getOperator())); } @@ -218,18 +211,20 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep private boolean castExprUsingUDFBridge(GenericUDF gUDF) { boolean castExpr = false; - if (gUDF != null && gUDF instanceof GenericUDFBridge) { + if (gUDF instanceof GenericUDFBridge) { String udfClassName = ((GenericUDFBridge) gUDF).getUdfClassName(); if (udfClassName != null) { int sp = udfClassName.lastIndexOf('.'); // TODO: add method to UDFBridge to say if it is a cast func - if (sp >= 0 & (sp + 1) < udfClassName.length()) { + if (sp >= 0 && (sp + 1) < udfClassName.length()) { udfClassName = udfClassName.substring(sp + 1); - if (udfClassName.equals("UDFToBoolean") || udfClassName.equals("UDFToByte") - || udfClassName.equals("UDFToDouble") || udfClassName.equals("UDFToInteger") - || udfClassName.equals("UDFToLong") || udfClassName.equals("UDFToShort") - || udfClassName.equals("UDFToFloat") || udfClassName.equals("UDFToString")) + if ("UDFToBoolean".equals(udfClassName) || "UDFToByte".equals(udfClassName) + || "UDFToDouble".equals(udfClassName) || "UDFToInteger".equals(udfClassName) + || "UDFToLong".equals(udfClassName) || "UDFToShort".equals(udfClassName) + || "UDFToFloat".equals(udfClassName) || "UDFToString".equals(udfClassName)) { castExpr = true; + } + } } } @@ -246,8 +241,8 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar) || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate) || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { - castExpr = m_cluster.getRexBuilder().makeAbstractCast( - TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()), + castExpr = cluster.getRexBuilder().makeAbstractCast( + TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()), childRexNodeLst.get(0)); } } @@ -258,23 +253,23 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { InputCtx ctxLookingFor = null; - if (m_inputCtxs.size() == 1) { - ctxLookingFor = m_inputCtxs.get(0); + if (inputCtxs.size() == 1) { + ctxLookingFor = inputCtxs.get(0); } else { String tableAlias = col.getTabAlias(); String colAlias = col.getColumn(); int noInp = 0; - for (InputCtx ic : m_inputCtxs) { - if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) { - if (ic.m_hiveRR.getPosition(colAlias) >= 0) { - ctxLookingFor = ic; - noInp++; - } + for (InputCtx ic : inputCtxs) { + if ((tableAlias == null || ic.hiveRR.hasTableAlias(tableAlias)) + && ic.hiveRR.getPosition(colAlias) >= 0) { + ctxLookingFor = ic; + noInp++; } } - if (noInp > 1) - throw new RuntimeException("Ambigous column mapping"); + if (noInp > 1) { + throw new RuntimeException("Ambiguous column mapping"); + } } return ctxLookingFor; @@ -282,16 +277,16 @@ private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException { InputCtx ic = getInputCtx(col); - int pos = ic.m_hiveNameToPosMap.get(col.getColumn()); - return m_cluster.getRexBuilder().makeInputRef( - ic.m_optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.m_offsetInOptiqSchema); + int pos = ic.hiveNameToPosMap.get(col.getColumn()); + return cluster.getRexBuilder().makeInputRef( + ic.optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.offsetInOptiqSchema); } - private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE), - MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticException { - RexBuilder rexBuilder = m_cluster.getRexBuilder(); + RexBuilder rexBuilder = cluster.getRexBuilder(); RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory(); PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo(); RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory); @@ -302,14 +297,14 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce Object value = ObjectInspectorUtils.copyToStandardJavaObject(coi.getWritableConstantValue(), coi); - RexNode optiqLiteral = null; + RexNode optiqLiteral; // TODO: Verify if we need to use ConstantObjectInspector to unwrap data switch (hiveTypeCategory) { case BOOLEAN: - optiqLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue()); + optiqLiteral = rexBuilder.makeLiteral((Boolean) value); break; case BYTE: - byte[] byteArray = new byte[] { (Byte) value }; + byte[] byteArray = {(Byte) value}; ByteString bs = new ByteString(byteArray); optiqLiteral = rexBuilder.makeBinaryLiteral(bs); break; @@ -353,7 +348,7 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce // will work... // An alternative would be to throw CboSemanticException and fall back // to no CBO. - RelDataType relType = m_cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, + RelDataType relType = cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length()); optiqLiteral = rexBuilder.makeExactLiteral(bd, relType); } @@ -365,13 +360,15 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce optiqLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), optiqDataType); break; case CHAR: - if (value instanceof HiveChar) + if (value instanceof HiveChar) { value = ((HiveChar) value).getValue(); + } optiqLiteral = rexBuilder.makeLiteral((String) value); break; case VARCHAR: - if (value instanceof HiveVarchar) + if (value instanceof HiveVarchar) { value = ((HiveVarchar) value).getValue(); + } optiqLiteral = rexBuilder.makeLiteral((String) value); break; case STRING: @@ -397,12 +394,12 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce } private RexNode createNullLiteral(ExprNodeDesc expr) throws OptiqSemanticException { - return m_cluster.getRexBuilder().makeNullLiteral( - TypeConverter.convert(expr.getTypeInfo(), m_cluster.getTypeFactory()).getSqlTypeName()); + return cluster.getRexBuilder().makeNullLiteral( + TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName()); } public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode, - List inputRels, LinkedHashMap relToHiveRR, + List inputRels, Map relToHiveRR, Map> relToHiveColNameOptiqPosMap, boolean flattenExpr) throws SemanticException { List inputCtxLst = new ArrayList(); @@ -414,6 +411,6 @@ public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprN offSet += r.getRowType().getFieldCount(); } - return (new RexNodeConverter(cluster, inputCtxLst, flattenExpr)).convert(joinCondnExprNode); + return new RexNodeConverter(cluster, inputCtxLst, flattenExpr).convert(joinCondnExprNode); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java index 31f906a..a342c3c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java @@ -66,15 +66,15 @@ public class SqlFunctionConverter { private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); - static final Map hiveToOptiq; - static final Map optiqToHiveToken; - static final Map reverseOperatorMap; + static final Map HIVE_TO_OPTIQ; + static final Map OPTIQ_TO_HIVE_TOKEN; + static final Map REVERSE_OPERATOR_MAP; static { StaticBlockBuilder builder = new StaticBlockBuilder(); - hiveToOptiq = ImmutableMap.copyOf(builder.hiveToOptiq); - optiqToHiveToken = ImmutableMap.copyOf(builder.optiqToHiveToken); - reverseOperatorMap = ImmutableMap.copyOf(builder.reverseOperatorMap); + HIVE_TO_OPTIQ = ImmutableMap.copyOf(builder.hiveToOptiq); + OPTIQ_TO_HIVE_TOKEN = ImmutableMap.copyOf(builder.optiqToHiveToken); + REVERSE_OPERATOR_MAP = ImmutableMap.copyOf(builder.reverseOperatorMap); } public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveUDF, @@ -97,12 +97,14 @@ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveU } public static GenericUDF getHiveUDF(SqlOperator op, RelDataType dt) { - String name = reverseOperatorMap.get(op); - if (name == null) + String name = REVERSE_OPERATOR_MAP.get(op); + if (name == null) { name = op.getName(); - FunctionInfo hFn = name != null ? FunctionRegistry.getFunctionInfo(name) : null; - if (hFn == null) + } + FunctionInfo hFn = name == null ? null : FunctionRegistry.getFunctionInfo(name); + if (hFn == null) { hFn = handleExplicitCast(op, dt); + } return hFn == null ? null : hFn.getGenericUDF(); } @@ -143,31 +145,31 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { FunctionRegistry.getFunctionInfo("decimal")); } else if (castType.equals(TypeInfoFactory.binaryTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("binary"); - } else throw new IllegalStateException("Unexpected type : " + - castType.getQualifiedName()); + } else { + throw new IllegalStateException("Unexpected type : " + + castType.getQualifiedName()); + } } return castUDF; } private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, FunctionInfo fi) { - SettableUDF udf = (SettableUDF)fi.getGenericUDF(); + SettableUDF udf = (SettableUDF) fi.getGenericUDF(); try { udf.setTypeInfo(ti); } catch (UDFArgumentException e) { throw new RuntimeException(e); } - return new FunctionInfo(fi.isNative(),fi.getDisplayName(),(GenericUDF)udf); + return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf); } // TODO: 1) handle Agg Func Name translation 2) is it correct to add func args // as child of func? public static ASTNode buildAST(SqlOperator op, List children) { - HiveToken hToken = optiqToHiveToken.get(op); + HiveToken hToken = OPTIQ_TO_HIVE_TOKEN.get(op); ASTNode node; - if (hToken != null) { - node = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); - } else { + if (hToken == null) { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTION, "TOK_FUNCTION"); if (op.kind != SqlKind.CAST) { if (op.kind == SqlKind.MINUS_PREFIX) { @@ -175,15 +177,16 @@ public static ASTNode buildAST(SqlOperator op, List children) { } else if (op.kind == SqlKind.PLUS_PREFIX) { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.PLUS, "PLUS"); } else { - if (op.getName().toUpperCase() - .equals(SqlStdOperatorTable.COUNT.getName()) - && children.size() == 0) { - node = (ASTNode) ParseDriver.adaptor.create( - HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); + if (op.getName().equalsIgnoreCase(SqlStdOperatorTable.COUNT.getName()) + && children.isEmpty()) { + node = + (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); } node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName())); } } + } else { + node = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); } for (ASTNode c : children) { @@ -199,8 +202,8 @@ public static ASTNode buildAST(SqlOperator op, List children) { */ public static ASTNode buildAST(SqlOperator op, List children, int i) { if (i + 1 < children.size()) { - HiveToken hToken = optiqToHiveToken.get(op); - ASTNode curNode = ((ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text)); + HiveToken hToken = OPTIQ_TO_HIVE_TOKEN.get(op); + ASTNode curNode = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text); ParseDriver.adaptor.addChild(curNode, children.get(i)); ParseDriver.adaptor.addChild(curNode, buildAST(op, children, i + 1)); return curNode; @@ -216,24 +219,25 @@ public static ASTNode buildAST(SqlOperator op, List children, int i) { private static String getName(GenericUDF hiveUDF) { String udfName = null; if (hiveUDF instanceof GenericUDFBridge) { - udfName = ((GenericUDFBridge) hiveUDF).getUdfName(); + udfName = hiveUDF.getUdfName(); } else { Class udfClass = hiveUDF.getClass(); Annotation udfAnnotation = udfClass.getAnnotation(Description.class); - if (udfAnnotation != null && udfAnnotation instanceof Description) { + if (udfAnnotation instanceof Description) { Description udfDescription = (Description) udfAnnotation; udfName = udfDescription.name(); if (udfName != null) { String[] aliases = udfName.split(","); - if (aliases.length > 0) + if (aliases.length > 0) { udfName = aliases[0]; + } } } if (udfName == null || udfName.isEmpty()) { udfName = hiveUDF.getClass().getName(); - int indx = udfName.lastIndexOf("."); + int indx = udfName.lastIndexOf('.'); if (indx >= 0) { indx += 1; udfName = udfName.substring(indx); @@ -246,9 +250,9 @@ private static String getName(GenericUDF hiveUDF) { /** This class is used to build immutable hashmaps in the static block above. */ private static class StaticBlockBuilder { - final Map hiveToOptiq = Maps.newHashMap(); - final Map optiqToHiveToken = Maps.newHashMap(); - final Map reverseOperatorMap = Maps.newHashMap(); + private final Map hiveToOptiq = Maps.newHashMap(); + private final Map optiqToHiveToken = Maps.newHashMap(); + private final Map reverseOperatorMap = Maps.newHashMap(); StaticBlockBuilder() { registerFunction("+", SqlStdOperatorTable.PLUS, hToken(HiveParser.PLUS, "+")); @@ -287,70 +291,71 @@ private static HiveToken hToken(int type, String text) { } public static class OptiqUDAF extends SqlAggFunction { - final ImmutableList m_argTypes; - final RelDataType m_retType; + private final ImmutableList argTypes; + private final RelDataType retType; public OptiqUDAF(String opName, SqlReturnTypeInference returnTypeInference, SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, ImmutableList argTypes, RelDataType retType) { super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference, operandTypeInference, operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); - m_argTypes = argTypes; - m_retType = retType; + this.argTypes = argTypes; + this.retType = retType; } @Override public List getParameterTypes(final RelDataTypeFactory typeFactory) { - return m_argTypes; + return argTypes; } @Override public RelDataType getReturnType(final RelDataTypeFactory typeFactory) { - return m_retType; + return retType; } } private static class OptiqUDFInfo { - private String m_udfName; - private SqlReturnTypeInference m_returnTypeInference; - private SqlOperandTypeInference m_operandTypeInference; - private SqlOperandTypeChecker m_operandTypeChecker; - private ImmutableList m_argTypes; - private RelDataType m_retType; + private String udfName; + private SqlReturnTypeInference returnTypeInference; + private SqlOperandTypeInference operandTypeInference; + private SqlOperandTypeChecker operandTypeChecker; + private ImmutableList argTypes; + private RelDataType retType; } private static OptiqUDFInfo getUDFInfo(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { OptiqUDFInfo udfInfo = new OptiqUDFInfo(); - udfInfo.m_udfName = hiveUdfName; - udfInfo.m_returnTypeInference = ReturnTypes.explicit(optiqRetType); - udfInfo.m_operandTypeInference = InferTypes.explicit(optiqArgTypes); - ImmutableList.Builder typeFamilyBuilder = new ImmutableList.Builder(); + udfInfo.udfName = hiveUdfName; + udfInfo.returnTypeInference = ReturnTypes.explicit(optiqRetType); + udfInfo.operandTypeInference = InferTypes.explicit(optiqArgTypes); + ImmutableList.Builder typeFamilyBuilder = + new ImmutableList.Builder(); for (RelDataType at : optiqArgTypes) { typeFamilyBuilder.add(Util.first(at.getSqlTypeName().getFamily(), SqlTypeFamily.ANY)); } - udfInfo.m_operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); + udfInfo.operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); - udfInfo.m_argTypes = ImmutableList. copyOf(optiqArgTypes); - udfInfo.m_retType = optiqRetType; + udfInfo.argTypes = ImmutableList.copyOf(optiqArgTypes); + udfInfo.retType = optiqRetType; return udfInfo; } - public static SqlOperator getOptiqFn(String hiveUdfName, - ImmutableList optiqArgTypes, RelDataType optiqRetType) throws OptiqSemanticException{ + public static SqlOperator getOptiqFn(String hiveUdfName, ImmutableList optiqArgTypes, + RelDataType optiqRetType) throws OptiqSemanticException { - if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) { + if (hiveUdfName != null && "<=>".equals(hiveUdfName.trim())) { // We can create Optiq IS_DISTINCT_FROM operator for this. But since our // join reordering algo cant handle this anyway there is no advantage of this. // So, bail out for now. throw new OptiqSemanticException("<=> is not yet supported for cbo."); } - SqlOperator optiqOp = hiveToOptiq.get(hiveUdfName); + SqlOperator optiqOp = HIVE_TO_OPTIQ.get(hiveUdfName); if (optiqOp == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqOp = new SqlFunction(uInf.m_udfName, SqlKind.OTHER_FUNCTION, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, + optiqOp = new SqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, + uInf.operandTypeInference, uInf.operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); } @@ -359,12 +364,12 @@ public static SqlOperator getOptiqFn(String hiveUdfName, public static SqlAggFunction getOptiqAggFn(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { - SqlAggFunction optiqAggFn = (SqlAggFunction) hiveToOptiq.get(hiveUdfName); + SqlAggFunction optiqAggFn = (SqlAggFunction) HIVE_TO_OPTIQ.get(hiveUdfName); if (optiqAggFn == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqAggFn = new OptiqUDAF(uInf.m_udfName, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, uInf.m_argTypes, uInf.m_retType); + optiqAggFn = new OptiqUDAF(uInf.udfName, uInf.returnTypeInference, + uInf.operandTypeInference, uInf.operandTypeChecker, uInf.argTypes, uInf.retType); } return optiqAggFn; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java index 2c30e9d..d916ce3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java @@ -50,12 +50,12 @@ import com.google.common.collect.ImmutableMap.Builder; import com.google.common.collect.Lists; -public class TypeConverter { - private static final Map optiqToHiveTypeNameMap; +public final class TypeConverter { + private static final Map OPTIQ_TO_HIVE_TYPE_NAME_MAP; // TODO: Handling of char[], varchar[], string... static { - Builder b = ImmutableMap. builder(); + Builder b = ImmutableMap.builder(); b.put(SqlTypeName.BOOLEAN.getName(), new HiveToken(HiveParser.TOK_BOOLEAN, "TOK_BOOLEAN")); b.put(SqlTypeName.TINYINT.getName(), new HiveToken(HiveParser.TOK_TINYINT, "TOK_TINYINT")); b.put(SqlTypeName.SMALLINT.getName(), new HiveToken(HiveParser.TOK_SMALLINT, "TOK_SMALLINT")); @@ -64,12 +64,13 @@ b.put(SqlTypeName.FLOAT.getName(), new HiveToken(HiveParser.TOK_FLOAT, "TOK_FLOAT")); b.put(SqlTypeName.DOUBLE.getName(), new HiveToken(HiveParser.TOK_DOUBLE, "TOK_DOUBLE")); b.put(SqlTypeName.DATE.getName(), new HiveToken(HiveParser.TOK_DATE, "TOK_DATE")); - b.put(SqlTypeName.TIMESTAMP.getName(), new HiveToken(HiveParser.TOK_TIMESTAMP, "TOK_TIMESTAMP")); + b.put(SqlTypeName.TIMESTAMP.getName(), + new HiveToken(HiveParser.TOK_TIMESTAMP, "TOK_TIMESTAMP")); b.put(SqlTypeName.BINARY.getName(), new HiveToken(HiveParser.TOK_BINARY, "TOK_BINARY")); - optiqToHiveTypeNameMap = b.build(); - }; + OPTIQ_TO_HIVE_TYPE_NAME_MAP = b.build(); + } - /*********************** Convert Hive Types To Optiq Types ***********************/ + /* Convert Hive Types To Optiq Types. */ public static RelDataType getType(RelOptCluster cluster, List cInfoLst) throws OptiqSemanticException { RexBuilder rexBuilder = cluster.getRexBuilder(); @@ -102,7 +103,7 @@ public static RelDataType getType(RelOptCluster cluster, RowResolver rr, } public static RelDataType convert(TypeInfo type, RelDataTypeFactory dtFactory) - throws OptiqSemanticException{ + throws OptiqSemanticException { RelDataType convertedType = null; switch (type.getCategory()) { @@ -205,15 +206,16 @@ public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory dtFact public static RelDataType convert(StructTypeInfo structType, final RelDataTypeFactory dtFactory) throws OptiqSemanticException { - List fTypes = new ArrayList(structType.getAllStructFieldTypeInfos().size()); + List fTypes = + new ArrayList(structType.getAllStructFieldTypeInfos().size()); for (TypeInfo ti : structType.getAllStructFieldTypeInfos()) { - fTypes.add(convert(ti,dtFactory)); + fTypes.add(convert(ti, dtFactory)); } return dtFactory.createStructType(fTypes, structType.getAllStructFieldNames()); } public static RelDataType convert(UnionTypeInfo unionType, RelDataTypeFactory dtFactory) - throws OptiqSemanticException{ + throws OptiqSemanticException { // Union type is not supported in Optiq. throw new OptiqSemanticException("Union type is not supported"); } @@ -282,10 +284,11 @@ public static TypeInfo convertPrimtiveType(RelDataType rType) { case DECIMAL: return TypeInfoFactory.getDecimalTypeInfo(rType.getPrecision(), rType.getScale()); case VARCHAR: - if (rType.getPrecision() == Integer.MAX_VALUE) + if (rType.getPrecision() == Integer.MAX_VALUE) { return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - else + } else { return TypeInfoFactory.getVarcharTypeInfo(rType.getPrecision()); + } case CHAR: return TypeInfoFactory.getCharTypeInfo(rType.getPrecision()); case OTHER: @@ -295,31 +298,29 @@ public static TypeInfo convertPrimtiveType(RelDataType rType) { } - /*********************** Convert Optiq Types To Hive Types ***********************/ + /* Convert Optiq Types To Hive Types */ public static HiveToken hiveToken(RelDataType optiqType) { - HiveToken ht = null; + HiveToken ht; switch (optiqType.getSqlTypeName()) { - case CHAR: { + case CHAR: ht = new HiveToken(HiveParser.TOK_CHAR, "TOK_CHAR", String.valueOf(optiqType.getPrecision())); - } break; - case VARCHAR: { - if (optiqType.getPrecision() == Integer.MAX_VALUE) - ht = new HiveToken(HiveParser.TOK_STRING, "TOK_STRING", String.valueOf(optiqType - .getPrecision())); - else - ht = new HiveToken(HiveParser.TOK_VARCHAR, "TOK_VARCHAR", String.valueOf(optiqType - .getPrecision())); - } + case VARCHAR: + if (optiqType.getPrecision() == Integer.MAX_VALUE) { + ht = new HiveToken(HiveParser.TOK_STRING, "TOK_STRING", + String.valueOf(optiqType.getPrecision())); + } else { + ht = new HiveToken(HiveParser.TOK_VARCHAR, "TOK_VARCHAR", + String.valueOf(optiqType.getPrecision())); + } break; - case DECIMAL: { + case DECIMAL: ht = new HiveToken(HiveParser.TOK_DECIMAL, "TOK_DECIMAL", String.valueOf(optiqType .getPrecision()), String.valueOf(optiqType.getScale())); - } break; default: - ht = optiqToHiveTypeNameMap.get(optiqType.getSqlTypeName().getName()); + ht = OPTIQ_TO_HIVE_TYPE_NAME_MAP.get(optiqType.getSqlTypeName().getName()); } return ht; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index 9c55379..ec1f22d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -352,39 +352,33 @@ public void setExpressionMap(Map expressionMap) { } - // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or - // not? - public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, - int outputColPos, int numColumns) throws SemanticException { - String tabAlias; - String colAlias; - String[] qualifiedColName; + // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or not? + public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, int outputColPos, + int numColumns) throws SemanticException { int i = 0; for (ColumnInfo cInfoFrmInput : rrToAddFrom.getRowSchema().getSignature()) { - if ( numColumns >= 0 && i == numColumns ) { + if (numColumns >= 0 && i == numColumns) { break; } - ColumnInfo newCI = null; - qualifiedColName = rrToAddFrom.getInvRslvMap().get( - cInfoFrmInput.getInternalName()); - tabAlias = qualifiedColName[0]; - colAlias = qualifiedColName[1]; + String[] qualifiedColName = rrToAddFrom.getInvRslvMap().get(cInfoFrmInput.getInternalName()); + String tabAlias = qualifiedColName[0]; + String colAlias = qualifiedColName[1]; - newCI = new ColumnInfo(cInfoFrmInput); - newCI.setInternalName(SemanticAnalyzer - .getColumnInternalName(outputColPos)); + ColumnInfo newCI = new ColumnInfo(cInfoFrmInput); + newCI.setInternalName(SemanticAnalyzer.getColumnInternalName(outputColPos)); outputColPos++; - if (rrToAddTo.get(tabAlias, colAlias) != null) { - LOG.debug("Found duplicate column alias in RR: " + rrToAddTo.get(tabAlias, colAlias)); - } else { + if (rrToAddTo.get(tabAlias, colAlias) == null) { rrToAddTo.put(tabAlias, colAlias, newCI); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Found duplicate column alias in RR: " + rrToAddTo.get(tabAlias, colAlias)); + } } - qualifiedColName = rrToAddFrom.getAlternateMappings(cInfoFrmInput - .getInternalName()); + qualifiedColName = rrToAddFrom.getAlternateMappings(cInfoFrmInput.getInternalName()); if (qualifiedColName != null) { tabAlias = qualifiedColName[0]; colAlias = qualifiedColName[1]; @@ -394,30 +388,25 @@ public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, } return outputColPos; - } + } public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, int outputColPos) throws SemanticException { return add(rrToAddTo, rrToAddFrom, outputColPos, -1); } - /** - * Return a new row resolver that is combination of left RR and right RR. - * The schema will be schema of left, schema of right - * - * @param leftRR - * @param rightRR - * @return - * @throws SemanticException - */ - public static RowResolver getCombinedRR(RowResolver leftRR, - RowResolver rightRR) throws SemanticException { - int outputColPos = 0; - - RowResolver combinedRR = new RowResolver(); - outputColPos = add(combinedRR, leftRR, outputColPos); - outputColPos = add(combinedRR, rightRR, outputColPos); - - return combinedRR; - } + /** + * Return a new row resolver that is combination of left RR and right RR. + * The schema will be schema of left, schema of right. + */ + public static RowResolver getCombinedRR(RowResolver leftRR, + RowResolver rightRR) throws SemanticException { + int outputColPos = 0; + + RowResolver combinedRR = new RowResolver(); + outputColPos = add(combinedRR, leftRR, outputColPos); + add(combinedRR, rightRR, outputColPos); + + return combinedRR; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 48d334a..fb44fbb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -37,7 +37,6 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -69,7 +68,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -238,7 +236,6 @@ import org.eigenbase.rel.AggregateRelBase; import org.eigenbase.rel.Aggregation; import org.eigenbase.rel.FilterRelBase; -import org.eigenbase.rel.InvalidRelException; import org.eigenbase.rel.JoinRelBase; import org.eigenbase.rel.JoinRelType; import org.eigenbase.rel.RelCollation; @@ -2707,7 +2704,7 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, @SuppressWarnings("nls") private Integer genColListRegex(String colRegex, String tabAlias, - ASTNode sel, ArrayList col_list, + ASTNode sel, List col_list, RowResolver input, Integer pos, RowResolver output, List aliases, boolean subQuery) throws SemanticException { @@ -3660,7 +3657,7 @@ private String recommendName(ExprNodeDesc exp, String colAlias) { * Class to store GenericUDAF related information. */ static class GenericUDAFInfo { - ArrayList convertedParameters; + List convertedParameters; GenericUDAFEvaluator genericUDAFEvaluator; TypeInfo returnType; } @@ -3679,7 +3676,7 @@ private String recommendName(ExprNodeDesc exp, String colAlias) { /** * Convert exprNodeDesc array to ObjectInspector array. */ - static ArrayList getWritableObjectInspector(ArrayList exprs) { + static ArrayList getWritableObjectInspector(List exprs) { ArrayList result = new ArrayList(); for (ExprNodeDesc expr : exprs) { result.add(expr.getWritableObjectInspector()); @@ -3704,10 +3701,10 @@ private String recommendName(ExprNodeDesc exp, String colAlias) { * for each GroupBy aggregation. */ static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, - ArrayList aggParameters, ASTNode aggTree, + List aggParameters, ASTNode aggTree, boolean isDistinct, boolean isAllColumns) throws SemanticException { - ArrayList originalParameterTypeInfos = + List originalParameterTypeInfos = getWritableObjectInspector(aggParameters); GenericUDAFEvaluator result = FunctionRegistry.getGenericUDAFEvaluator( aggName, originalParameterTypeInfos, isDistinct, isAllColumns); @@ -3734,7 +3731,7 @@ static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, * when the UDAF is not found or has problems. */ static GenericUDAFInfo getGenericUDAFInfo(GenericUDAFEvaluator evaluator, - GenericUDAFEvaluator.Mode emode, ArrayList aggParameters) + GenericUDAFEvaluator.Mode emode, List aggParameters) throws SemanticException { GenericUDAFInfo r = new GenericUDAFInfo(); @@ -9792,7 +9789,7 @@ private Operator genLateralViewPlan(QB qb, Operator op, ASTNode lateralViewTree) * the same order as in the dest row resolver */ private void LVmergeRowResolvers(RowResolver source, RowResolver dest, - Map colExprMap, ArrayList outputInternalColNames) { + Map colExprMap, List outputInternalColNames) { for (ColumnInfo c : source.getColumnInfos()) { String internalName = getColumnInternalName(outputInternalColNames.size()); outputInternalColNames.add(internalName); @@ -10507,7 +10504,7 @@ public RowResolver getRowResolver(Operator opt) { * Add default properties for table property. If a default parameter exists * in the tblProp, the value in tblProp will be kept. * - * @param table + * @param tblProp * property map * @return Modified table property map */ @@ -12212,11 +12209,17 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { // 1. If top level QB is query then everything below it must also be Query // 2. Nested Subquery will return false for qbToChk.getIsQuery() if ((!topLevelQB || qbToChk.getIsQuery()) - && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict")) - && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST)) - && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy() - && !queryProperties.hasSortBy() && !queryProperties.hasPTF() - && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery() + && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) + || "nonstrict".equalsIgnoreCase(conf.getVar(ConfVars.HIVEMAPREDMODE))) + && (!topLevelQB + || (queryProperties.getJoinCount() > 1) + || conf.getBoolVar(ConfVars.HIVE_IN_TEST)) + && !queryProperties.hasClusterBy() + && !queryProperties.hasDistributeBy() + && !queryProperties.hasSortBy() + && !queryProperties.hasPTF() + && !queryProperties.usesScript() + && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) { runOptiqPlanner = true; } else { @@ -12231,39 +12234,36 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { private RelOptSchema relOptSchema; private SemanticException semanticException; private Map partitionCache; - private final AtomicInteger noColsMissingStats = new AtomicInteger(0); - List topLevelFieldSchema; + private final AtomicInteger noColsMissingStats = new AtomicInteger(0); + private List topLevelFieldSchema; // TODO: Do we need to keep track of RR, ColNameToPosMap for every op or // just last one. - LinkedHashMap relToHiveRR = new LinkedHashMap(); - LinkedHashMap> relToHiveColNameOptiqPosMap = new LinkedHashMap>(); + private final Map relToHiveRR = new LinkedHashMap(); + private final Map> relToHiveColNameOptiqPosMap = + new LinkedHashMap>(); private ASTNode getOptimizedAST(Map partitionCache) throws SemanticException { - ASTNode optiqOptimizedAST = null; - RelNode optimizedOptiqPlan = null; this.partitionCache = partitionCache; + RelNode optimizedOptiqPlan; try { optimizedOptiqPlan = Frameworks.withPlanner(this, - Frameworks.newConfigBuilder().typeSystem(new HiveTypeSystemImpl()).build()); + Frameworks.newConfigBuilder().typeSystem(new HiveTypeSystemImpl()).build()); } catch (Exception e) { - if (semanticException != null) - throw semanticException; - else + if (semanticException == null) { throw new RuntimeException(e); + } else { + throw semanticException; + } } - optiqOptimizedAST = ASTConverter.convert(optimizedOptiqPlan, topLevelFieldSchema); - return optiqOptimizedAST; + return ASTConverter.convert(optimizedOptiqPlan, topLevelFieldSchema); } @Override public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) { - RelNode optiqGenPlan = null; - RelNode optiqPreCboPlan = null; - RelNode optiqOptimizedPlan = null; /* * recreate cluster, so that it picks up the additional traitDef @@ -12276,6 +12276,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu this.cluster = cluster; this.relOptSchema = relOptSchema; + RelNode optiqGenPlan; try { optiqGenPlan = genLogicalPlan(qb, true); topLevelFieldSchema = convertRowSchemaToResultSetSchema(relToHiveRR.get(optiqGenPlan), @@ -12285,18 +12286,18 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu throw new RuntimeException(e); } - optiqPreCboPlan = applyPreCBOTransforms(optiqGenPlan, HiveDefaultRelMetadataProvider.INSTANCE); + RelNode optiqPreCboPlan = + applyPreCBOTransforms(optiqGenPlan, HiveDefaultRelMetadataProvider.INSTANCE); List list = Lists.newArrayList(); list.add(HiveDefaultRelMetadataProvider.INSTANCE); RelTraitSet desiredTraits = cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); - HepProgram hepPgm = null; HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP) .addRuleInstance(new ConvertMultiJoinRule(HiveJoinRel.class)); hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoinRel.HIVE_JOIN_FACTORY, HiveProjectRel.DEFAULT_PROJECT_FACTORY, HiveFilterRel.DEFAULT_FILTER_FACTORY)); - hepPgm = hepPgmBldr.build(); + HepProgram hepPgm = hepPgmBldr.build(); HepPlanner hepPlanner = new HepPlanner(hepPgm); hepPlanner.registerMetadataProviders(list); @@ -12310,7 +12311,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu } hepPlanner.setRoot(rootRel); - optiqOptimizedPlan = hepPlanner.findBestExp(); + RelNode optiqOptimizedPlan = hepPlanner.findBestExp(); if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { LOG.debug("CBO Planning details:\n"); @@ -12345,11 +12346,13 @@ public RelNode applyPreCBOTransforms(RelNode basePlan, RelMetadataProvider mdPro JoinRelBase.class, HiveFilterRel.DEFAULT_FILTER_FACTORY), // TODO: Enable it after OPTIQ-407 is fixed //RemoveTrivialProjectRule.INSTANCE, - new HivePartitionPrunerRule(SemanticAnalyzer.this.conf)); + new HivePartitionPrunerRule(conf)); - RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProjectRel.DEFAULT_PROJECT_FACTORY, - HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveJoinRel.HIVE_JOIN_FACTORY, RelFactories.DEFAULT_SEMI_JOIN_FACTORY, - HiveSortRel.HIVE_SORT_REL_FACTORY, HiveAggregateRel.HIVE_AGGR_REL_FACTORY, HiveUnionRel.UNION_REL_FACTORY); + RelFieldTrimmer fieldTrimmer = + new RelFieldTrimmer(null, HiveProjectRel.DEFAULT_PROJECT_FACTORY, + HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveJoinRel.HIVE_JOIN_FACTORY, + RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSortRel.HIVE_SORT_REL_FACTORY, + HiveAggregateRel.HIVE_AGGR_REL_FACTORY, HiveUnionRel.UNION_REL_FACTORY); basePlan = fieldTrimmer.trim(basePlan); basePlan = hepPlan(basePlan, true, mdProvider, @@ -12363,15 +12366,15 @@ public RelNode applyPreCBOTransforms(RelNode basePlan, RelMetadataProvider mdPro private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, RelOptRule... rules) { - RelNode optimizedRelNode = basePlan; HepProgramBuilder programBuilder = new HepProgramBuilder(); if (followPlanChanges) { programBuilder.addMatchOrder(HepMatchOrder.TOP_DOWN); programBuilder = programBuilder.addRuleCollection(ImmutableList.copyOf(rules)); } else { // TODO: Should this be also TOP_DOWN? - for (RelOptRule r : rules) + for (RelOptRule r : rules) { programBuilder.addRuleInstance(r); + } } HepPlanner planner = new HepPlanner(programBuilder.build()); @@ -12383,22 +12386,20 @@ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, new CachingRelMetadataProvider(chainedProvider, planner)); planner.setRoot(basePlan); - optimizedRelNode = planner.findBestExp(); - return optimizedRelNode; + return planner.findBestExp(); } @SuppressWarnings("nls") private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode leftRel, String rightalias, RelNode rightRel) throws SemanticException { - HiveUnionRel unionRel = null; // 1. Get Row Resolvers, Column map for original left and right input of // Union Rel - RowResolver leftRR = this.relToHiveRR.get(leftRel); - RowResolver rightRR = this.relToHiveRR.get(rightRel); - HashMap leftmap = leftRR.getFieldMap(leftalias); - HashMap rightmap = rightRR.getFieldMap(rightalias); + RowResolver leftRR = relToHiveRR.get(leftRel); + RowResolver rightRR = relToHiveRR.get(rightRel); + Map leftmap = leftRR.getFieldMap(leftalias); + Map rightmap = rightRR.getFieldMap(rightalias); // 2. Validate that Union is feasible according to Hive (by using type // info from RR) @@ -12464,31 +12465,29 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode List leftRowDT = leftRel.getRowType().getFieldList(); List rightRowDT = rightRel.getRowType().getFieldList(); - RelDataType leftFieldDT; - RelDataType rightFieldDT; - RelDataType unionFieldDT; for (int i = 0; i < leftRowDT.size(); i++) { - leftFieldDT = leftRowDT.get(i).getType(); - rightFieldDT = rightRowDT.get(i).getType(); - if (!leftFieldDT.equals(rightFieldDT)) { - unionFieldDT = TypeConverter.convert(unionoutRR.getColumnInfos().get(i).getType(), - cluster.getTypeFactory()); + RelDataType leftFieldDT = leftRowDT.get(i).getType(); + RelDataType rightFieldDT = rightRowDT.get(i).getType(); + if (leftFieldDT.equals(rightFieldDT)) { + leftProjs.add(cluster.getRexBuilder() + .ensureType(leftFieldDT, cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); + rightProjs.add(cluster.getRexBuilder() + .ensureType(rightFieldDT, cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); + } else { + RelDataType unionFieldDT = + TypeConverter.convert(unionoutRR.getColumnInfos().get(i).getType(), + cluster.getTypeFactory()); if (!unionFieldDT.equals(leftFieldDT)) { leftNeedsTypeCast = true; } - leftProjs.add(cluster.getRexBuilder().ensureType(unionFieldDT, - cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); + leftProjs.add(cluster.getRexBuilder() + .ensureType(unionFieldDT, cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); if (!unionFieldDT.equals(rightFieldDT)) { rightNeedsTypeCast = true; } - rightProjs.add(cluster.getRexBuilder().ensureType(unionFieldDT, - cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); - } else { - leftProjs.add(cluster.getRexBuilder().ensureType(leftFieldDT, - cluster.getRexBuilder().makeInputRef(leftFieldDT, i), true)); - rightProjs.add(cluster.getRexBuilder().ensureType(rightFieldDT, - cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); + rightProjs.add(cluster.getRexBuilder() + .ensureType(unionFieldDT, cluster.getRexBuilder().makeInputRef(rightFieldDT, i), true)); } } @@ -12506,55 +12505,54 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode } // 6. Construct Union Rel - ImmutableList.Builder bldr = new ImmutableList.Builder(); + ImmutableList.Builder bldr = new ImmutableList.Builder(); bldr.add(unionLeftInput); bldr.add(unionRightInput); - unionRel = new HiveUnionRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), - bldr.build()); + RelNode unionRel = + new HiveUnionRel(cluster, TraitsUtil.getDefaultTraitSet(cluster), bldr.build()); relToHiveRR.put(unionRel, unionoutRR); - relToHiveColNameOptiqPosMap.put(unionRel, - this.buildHiveToOptiqColumnMap(unionoutRR, unionRel)); + relToHiveColNameOptiqPosMap.put(unionRel, buildHiveToOptiqColumnMap(unionoutRR, unionRel)); return unionRel; } private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJoinType, ASTNode joinCond) throws SemanticException { - RelNode joinRel = null; // 1. construct the RowResolver for the new Join Node by combining row // resolvers from left, right - RowResolver leftRR = this.relToHiveRR.get(leftRel); - RowResolver rightRR = this.relToHiveRR.get(rightRel); - RowResolver joinRR = null; + RowResolver leftRR = relToHiveRR.get(leftRel); + RowResolver rightRR = relToHiveRR.get(rightRel); + RowResolver joinRR; - if (hiveJoinType != JoinType.LEFTSEMI) { - joinRR = RowResolver.getCombinedRR(leftRR, rightRR); - } else { + if (hiveJoinType == JoinType.LEFTSEMI) { joinRR = new RowResolver(); RowResolver.add(joinRR, leftRR, 0); + } else { + joinRR = RowResolver.getCombinedRR(leftRR, rightRR); } // 2. Construct ExpressionNodeDesc representing Join Condition - RexNode optiqJoinCond = null; - if (joinCond != null) { + RexNode optiqJoinCond; + if (joinCond == null) { + optiqJoinCond = cluster.getRexBuilder().makeLiteral(true); + } else { JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); - Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, - jCtx); - if (jCtx.getError() != null) - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(jCtx.getErrorSrcNode(), - jCtx.getError())); + Map + exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, jCtx); + if (jCtx.getError() != null) { + throw new SemanticException( + generateErrorMessage(jCtx.getErrorSrcNode(), jCtx.getError())); + } ExprNodeDesc joinCondnExprNode = exprNodes.get(joinCond); List inputRels = new ArrayList(); inputRels.add(leftRel); inputRels.add(rightRel); - optiqJoinCond = RexNodeConverter.convert(cluster, joinCondnExprNode, inputRels, - relToHiveRR, relToHiveColNameOptiqPosMap, false); - } else { - optiqJoinCond = cluster.getRexBuilder().makeLiteral(true); + optiqJoinCond = RexNodeConverter.convert(cluster, joinCondnExprNode, inputRels, relToHiveRR, + relToHiveColNameOptiqPosMap, false); } // 3. Validate that join condition is legal (i.e no function refering to @@ -12585,6 +12583,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ break; } + RelNode joinRel; if (leftSemiJoin) { List sysFieldList = new ArrayList(); List leftJoinKeys = new ArrayList(); @@ -12598,7 +12597,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ + nonEquiConds); } - RelNode[] inputRels = new RelNode[] { leftRel, rightRel }; + RelNode[] inputRels = {leftRel, rightRel}; final List leftKeys = new ArrayList(); final List rightKeys = new ArrayList(); optiqJoinCond = HiveOptiqUtil.projectNonColumnEquiConditions( @@ -12613,7 +12612,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ leftSemiJoin); } // 5. Add new JoinRel & its RR to the maps - relToHiveColNameOptiqPosMap.put(joinRel, this.buildHiveToOptiqColumnMap(joinRR, joinRel)); + relToHiveColNameOptiqPosMap.put(joinRel, buildHiveToOptiqColumnMap(joinRR, joinRel)); relToHiveRR.put(joinRel, joinRR); return joinRel; @@ -12622,28 +12621,23 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ /** * Generate Join Logical Plan Relnode by walking through the join AST. * - * @param qb * @param aliasToRel * Alias(Table/Relation alias) to RelNode; only read and not * written in to by this method - * @return - * @throws SemanticException */ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map aliasToRel) throws SemanticException { - RelNode leftRel = null; - RelNode rightRel = null; - JoinType hiveJoinType = null; if (joinParseTree.getToken().getType() == HiveParser.TOK_UNIQUEJOIN) { - String msg = String.format("UNIQUE JOIN is currently not supported in CBO," - + " turn off cbo to use UNIQUE JOIN."); + String msg = "UNIQUE JOIN is currently not supported in CBO," + + " turn off cbo to use UNIQUE JOIN."; LOG.debug(msg); throw new OptiqSemanticException(msg); } // 1. Determine Join Type // TODO: What about TOK_CROSSJOIN, TOK_MAPJOIN + JoinType hiveJoinType; switch (joinParseTree.getToken().getType()) { case HiveParser.TOK_LEFTOUTERJOIN: hiveJoinType = JoinType.LEFTOUTER; @@ -12664,6 +12658,7 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // 2. Get Left Table Alias ASTNode left = (ASTNode) joinParseTree.getChild(0); + RelNode leftRel = null; if ((left.getToken().getType() == HiveParser.TOK_TABREF) || (left.getToken().getType() == HiveParser.TOK_SUBQUERY) || (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { @@ -12674,18 +12669,19 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) // guranteed to have an lias here: check done in processJoin - leftTableAlias = (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier(left - .getChild(1).getText().toLowerCase()) - : leftTableAlias; + leftTableAlias = + (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier( + left.getChild(1).getText().toLowerCase()) : leftTableAlias; leftRel = aliasToRel.get(leftTableAlias); } else if (isJoinToken(left)) { leftRel = genJoinLogicalPlan(left, aliasToRel); } else { - assert (false); + assert false; } // 3. Get Right Table Alias ASTNode right = (ASTNode) joinParseTree.getChild(1); + RelNode rightRel = null; if ((right.getToken().getType() == HiveParser.TOK_TABREF) || (right.getToken().getType() == HiveParser.TOK_SUBQUERY) || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { @@ -12696,12 +12692,12 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) // guranteed to have an lias here: check done in processJoin - rightTableAlias = (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier(right - .getChild(1).getText().toLowerCase()) - : rightTableAlias; + rightTableAlias = + (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) ? unescapeIdentifier( + right.getChild(1).getText().toLowerCase()) : rightTableAlias; rightRel = aliasToRel.get(rightTableAlias); } else { - assert (false); + assert false; } // 4. Get Join Condn @@ -12713,16 +12709,16 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticException { RowResolver rr = new RowResolver(); - HiveTableScanRel tableRel = null; + HiveTableScanRel tableRel; try { // 1. If the table has a Sample specified, bail from Optiq path. - if ( qb.getParseInfo().getTabSample(tableAlias) != null || - SemanticAnalyzer.this.nameToSplitSample.containsKey(tableAlias)) { + if (qb.getParseInfo().getTabSample(tableAlias) != null || nameToSplitSample.containsKey( + tableAlias)) { String msg = String.format("Table Sample specified for %s." + - " Currently we don't support Table Sample clauses in CBO," + - " turn off cbo for queries on tableSamples.", tableAlias); + " Currently we don't support Table Sample clauses in CBO," + + " turn off cbo for queries on tableSamples.", tableAlias); LOG.debug(msg); throw new OptiqSemanticException(msg); } @@ -12740,37 +12736,34 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc List fields = rowObjectInspector.getAllStructFieldRefs(); ColumnInfo colInfo; String colName; - ArrayList cInfoLst = new ArrayList(); - for (int i = 0; i < fields.size(); i++) { - colName = fields.get(i).getFieldName(); - colInfo = new ColumnInfo( - fields.get(i).getFieldName(), - TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()), - tableAlias, false); - colInfo.setSkewedCol((isSkewedCol(tableAlias, qb, colName)) ? true : false); + List cInfoLst = new ArrayList(); + for (StructField field : fields) { + colName = field.getFieldName(); + colInfo = new ColumnInfo(field.getFieldName(), + TypeInfoUtils.getTypeInfoFromObjectInspector(field.getFieldObjectInspector()), + tableAlias, false); + colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); rr.put(tableAlias, colName, colInfo); cInfoLst.add(colInfo); } // TODO: Fix this - ArrayList nonPartitionColumns = new ArrayList(cInfoLst); - ArrayList partitionColumns = new ArrayList(); + List nonPartitionColumns = new ArrayList(cInfoLst); + List partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema part_col : tab.getPartCols()) { - colName = part_col.getName(); + for (FieldSchema partCol : tab.getPartCols()) { + colName = partCol.getName(); colInfo = new ColumnInfo(colName, - TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), tableAlias, true); + TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), tableAlias, true); rr.put(tableAlias, colName, colInfo); cInfoLst.add(colInfo); partitionColumns.add(colInfo); } // 3.3 Add column info corresponding to virtual columns - Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); - while (vcs.hasNext()) { - VirtualColumn vc = vcs.next(); - colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true, - vc.getIsHidden()); + for (VirtualColumn vc : VirtualColumn.getRegistry(conf)) { + colInfo = + new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true, vc.getIsHidden()); rr.put(tableAlias, vc.getName(), colInfo); cInfoLst.add(colInfo); } @@ -12780,10 +12773,11 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 4. Build RelOptAbstractTable String fullyQualifiedTabName = tab.getDbName(); - if (fullyQualifiedTabName != null && !fullyQualifiedTabName.isEmpty()) - fullyQualifiedTabName = fullyQualifiedTabName + "." + tab.getTableName(); - else + if (fullyQualifiedTabName == null || fullyQualifiedTabName.isEmpty()) { fullyQualifiedTabName = tab.getTableName(); + } else { + fullyQualifiedTabName = fullyQualifiedTabName + "." + tab.getTableName(); + } RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, tableAlias, rowType, tab, nonPartitionColumns, partitionColumns, conf, partitionCache, noColsMissingStats); @@ -12796,12 +12790,10 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ImmutableMap hiveToOptiqColMap = buildHiveToOptiqColumnMap(rr, tableRel); relToHiveRR.put(tableRel, rr); relToHiveColNameOptiqPosMap.put(tableRel, hiveToOptiqColMap); + } catch (SemanticException e) { + throw e; } catch (Exception e) { - if (e instanceof SemanticException) { - throw (SemanticException) e; - } else { - throw (new RuntimeException(e)); - } + throw new RuntimeException(e); } return tableRel; @@ -12817,13 +12809,13 @@ private RelNode genFilterRelNode(ASTNode filterExpr, RelNode srcRel) throws Sema // for such queries, its an arcane corner case, not worth of adding that complexity. throw new OptiqSemanticException("Filter expression with non-boolean return type."); } - ImmutableMap hiveColNameOptiqPosMap = this.relToHiveColNameOptiqPosMap + ImmutableMap hiveColNameOptiqPosMap = relToHiveColNameOptiqPosMap .get(srcRel); RexNode convertedFilterExpr = new RexNodeConverter(cluster, srcRel.getRowType(), hiveColNameOptiqPosMap, 0, true).convert(filterCondn); RelNode filterRel = new HiveFilterRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), srcRel, convertedFilterExpr); - this.relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); + relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); relToHiveRR.put(filterRel, relToHiveRR.get(srcRel)); relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); @@ -12846,7 +12838,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, */ int numSrcColumns = srcRel.getRowType().getFieldCount(); List subQueriesInOriginalTree = SubQueryUtils.findSubQueries(searchCond); - if (subQueriesInOriginalTree.size() > 0) { + if (!subQueriesInOriginalTree.isEmpty()) { /* * Restriction.9.m :: disallow nested SubQuery expressions. @@ -12886,10 +12878,10 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QBSubQuery subQuery = SubQueryUtils.buildSubQuery(qb.getId(), sqIdx, subQueryAST, originalSubQueryAST, ctx); - if (!forHavingClause) { - qb.setWhereClauseSubQueryPredicate(subQuery); - } else { + if (forHavingClause) { qb.setHavingClauseSubQueryPredicate(subQuery); + } else { + qb.setWhereClauseSubQueryPredicate(subQuery); } String havingInputAlias = null; @@ -12903,8 +12895,8 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true); qbSQ.setSubQueryDef(subQuery.getSubQuery()); - Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1); + Phase1Ctx ctx1 = initPhase1Ctx(); + doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx1); getMetaData(qbSQ); RelNode subQueryRelNode = genLogicalPlan(qbSQ, false); aliasToRel.put(subQuery.getAlias(), subQueryRelNode); @@ -12916,7 +12908,8 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, */ if (subQuery.getOperator().getType() != SubQueryType.EXISTS && subQuery.getOperator().getType() != SubQueryType.NOT_EXISTS - && sqRR.getColumnInfos().size() - subQuery.getNumOfCorrelationExprsAddedToSQSelect() > 1) { + && sqRR.getColumnInfos().size() - subQuery.getNumOfCorrelationExprsAddedToSQSelect() + > 1) { throw new SemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(subQueryAST, "SubQuery can contain only 1 item in Select List.")); } @@ -12929,12 +12922,12 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, if (subQuery.getNotInCheck() != null) { QBSubQuery.NotInCheck notInCheck = subQuery.getNotInCheck(); notInCheck.setSQRR(sqRR); - QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); - qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery()); - ctx_1 = initPhase1Ctx(); - doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1); - getMetaData(qbSQ_nic); - RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false); + QB qbSQNic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); + qbSQNic.setSubQueryDef(notInCheck.getSubQuery()); + ctx1 = initPhase1Ctx(); + doPhase1(notInCheck.getSubQueryAST(), qbSQNic, ctx1); + getMetaData(qbSQNic); + RelNode subQueryNICRelNode = genLogicalPlan(qbSQNic, false); aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode); srcRel = genJoinRelNode(srcRel, subQueryNICRelNode, // set explicitly to inner until we figure out SemiJoin use @@ -12991,8 +12984,8 @@ private RelNode projectLeftOuterSide(RelNode srcRel, int numColumns) throws Sema HiveRel selRel = HiveProjectRel.create(srcRel, optiqColLst, oFieldNames); - this.relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(oRR, selRel)); - this.relToHiveRR.put(selRel, oRR); + relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(oRR, selRel)); + relToHiveRR.put(selRel, oRR); return selRel; } @@ -13014,39 +13007,36 @@ private RelNode genFilterLogicalPlan(QB qb, RelNode srcRel, Map * Class to store GenericUDAF related information. */ private class AggInfo { - private final List m_aggParams; - private final TypeInfo m_returnType; - private final String m_udfName; - private final boolean m_distinct; + private final List aggParams; + private final TypeInfo returnType; + private final String udfName; + private final boolean distinct; private AggInfo(List aggParams, TypeInfo returnType, String udfName, boolean isDistinct) { - m_aggParams = aggParams; - m_returnType = returnType; - m_udfName = udfName; - m_distinct = isDistinct; + this.aggParams = aggParams; + this.returnType = returnType; + this.udfName = udfName; + distinct = isDistinct; } } private AggregateCall convertGBAgg(AggInfo agg, RelNode input, List gbChildProjLst, - RexNodeConverter converter, HashMap rexNodeToPosMap, + RexNodeConverter converter, Map rexNodeToPosMap, Integer childProjLstIndx) throws SemanticException { // 1. Get agg fn ret type in Optiq - RelDataType aggFnRetType = TypeConverter.convert(agg.m_returnType, - this.cluster.getTypeFactory()); + RelDataType aggFnRetType = TypeConverter.convert(agg.returnType, cluster.getTypeFactory()); // 2. Convert Agg Fn args and type of args to Optiq // TODO: Does HQL allows expressions as aggregate args or can it only be // projections from child? - Integer inputIndx; List argList = new ArrayList(); - RexNode rexNd = null; - RelDataTypeFactory dtFactory = this.cluster.getTypeFactory(); + RelDataTypeFactory dtFactory = cluster.getTypeFactory(); ImmutableList.Builder aggArgRelDTBldr = new ImmutableList.Builder(); - for (ExprNodeDesc expr : agg.m_aggParams) { - rexNd = converter.convert(expr); - inputIndx = rexNodeToPosMap.get(rexNd.toString()); + for (ExprNodeDesc expr : agg.aggParams) { + RexNode rexNd = converter.convert(expr); + Integer inputIndx = rexNodeToPosMap.get(rexNd.toString()); if (inputIndx == null) { gbChildProjLst.add(rexNd); rexNodeToPosMap.put(rexNd.toString(), childProjLstIndx); @@ -13061,26 +13051,24 @@ private AggregateCall convertGBAgg(AggInfo agg, RelNode input, List gbC // 3. Get Aggregation FN from Optiq given name, ret type and input arg // type - final Aggregation aggregation = SqlFunctionConverter.getOptiqAggFn(agg.m_udfName, + final Aggregation aggregation = SqlFunctionConverter.getOptiqAggFn(agg.udfName, aggArgRelDTBldr.build(), aggFnRetType); - return new AggregateCall(aggregation, agg.m_distinct, argList, aggFnRetType, null); + return new AggregateCall(aggregation, agg.distinct, argList, aggFnRetType, null); } private RelNode genGBRelNode(List gbExprs, List aggInfoLst, RelNode srcRel) throws SemanticException { - RowResolver gbInputRR = this.relToHiveRR.get(srcRel); - ImmutableMap posMap = this.relToHiveColNameOptiqPosMap.get(srcRel); - RexNodeConverter converter = new RexNodeConverter(this.cluster, srcRel.getRowType(), + ImmutableMap posMap = relToHiveColNameOptiqPosMap.get(srcRel); + RexNodeConverter converter = new RexNodeConverter(cluster, srcRel.getRowType(), posMap, 0, false); final List gbChildProjLst = Lists.newArrayList(); - final HashMap rexNodeToPosMap = new HashMap(); + final Map rexNodeToPosMap = new HashMap(); final BitSet groupSet = new BitSet(); Integer gbIndx = 0; - RexNode rnd; for (ExprNodeDesc key : gbExprs) { - rnd = converter.convert(key); + RexNode rnd = converter.convert(key); gbChildProjLst.add(rnd); groupSet.set(gbIndx); rexNodeToPosMap.put(rnd.toString(), gbIndx); @@ -13088,7 +13076,6 @@ private RelNode genGBRelNode(List gbExprs, List aggInfoLs } List aggregateCalls = Lists.newArrayList(); - int i = aggInfoLst.size(); for (AggInfo agg : aggInfoLst) { aggregateCalls.add(convertGBAgg(agg, srcRel, gbChildProjLst, converter, rexNodeToPosMap, gbChildProjLst.size())); @@ -13097,32 +13084,25 @@ private RelNode genGBRelNode(List gbExprs, List aggInfoLs if (gbChildProjLst.isEmpty()) { // This will happen for count(*), in such cases we arbitarily pick // first element from srcRel - gbChildProjLst.add(this.cluster.getRexBuilder().makeInputRef(srcRel, 0)); + gbChildProjLst.add(cluster.getRexBuilder().makeInputRef(srcRel, 0)); } RelNode gbInputRel = HiveProjectRel.create(srcRel, gbChildProjLst, null); - HiveRel aggregateRel = null; - try { - aggregateRel = new HiveAggregateRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), - gbInputRel, groupSet, aggregateCalls); - } catch (InvalidRelException e) { - throw new SemanticException(e); - } - - return aggregateRel; + return new HiveAggregateRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), gbInputRel, + groupSet, aggregateCalls); } private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, RowResolver gByInputRR, RowResolver gByRR) { if (gByExpr.getType() == HiveParser.DOT && gByExpr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL) { - String tab_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getChild(0) + String tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getChild(0) .getText()); - String col_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(1).getText()); - gByRR.put(tab_alias, col_alias, colInfo); + String colAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(1).getText()); + gByRR.put(tabAlias, colAlias, colInfo); } else if (gByExpr.getType() == HiveParser.TOK_TABLE_OR_COL) { - String col_alias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getText()); - String tab_alias = null; + String colAlias = BaseSemanticAnalyzer.unescapeIdentifier(gByExpr.getChild(0).getText()); + String tabAlias = null; /* * If the input to the GBy has a tab alias for the column, then add an * entry based on that tab_alias. For e.g. this query: select b.x, @@ -13131,19 +13111,18 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, * ancestor before any GBy/ReduceSinks added for the GBY operation. */ try { - ColumnInfo pColInfo = gByInputRR.get(tab_alias, col_alias); - tab_alias = pColInfo == null ? null : pColInfo.getTabAlias(); + ColumnInfo pColInfo = gByInputRR.get(tabAlias, colAlias); + tabAlias = pColInfo == null ? null : pColInfo.getTabAlias(); } catch (SemanticException se) { } - gByRR.put(tab_alias, col_alias, colInfo); + gByRR.put(tabAlias, colAlias, colInfo); } } private void addToGBExpr(RowResolver groupByOutputRowResolver, RowResolver groupByInputRowResolver, ASTNode grpbyExpr, ExprNodeDesc grpbyExprNDesc, List gbExprNDescLst, List outputColumnNames) { - // TODO: Should we use grpbyExprNDesc.getTypeInfo()? what if expr is - // UDF + // TODO: Should we use grpbyExprNDesc.getTypeInfo()? what if expr is UDF int i = gbExprNDescLst.size(); String field = getColumnInternalName(i); outputColumnNames.add(field); @@ -13158,10 +13137,9 @@ private void addToGBExpr(RowResolver groupByOutputRowResolver, private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver inputRR) throws SemanticException { - AggInfo aInfo = null; // 1 Convert UDAF Params to ExprNodeDesc - ArrayList aggParameters = new ArrayList(); + List aggParameters = new ArrayList(); for (int i = 1; i <= aggFnLstArgIndx; i++) { ASTNode paraExpr = (ASTNode) aggAst.getChild(i); ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, inputRR); @@ -13179,10 +13157,11 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver // 3.2 Rank functions type is 'int'/'double' if (FunctionRegistry.isRankingFunction(aggName)) { - if (aggName.equalsIgnoreCase("percent_rank")) + if ("percent_rank".equalsIgnoreCase(aggName)) { udafRetType = TypeInfoFactory.doubleTypeInfo; - else + } else { udafRetType = TypeInfoFactory.intTypeInfo; + } } else { // 3.3 Try obtaining UDAF evaluators to determine the ret type try { @@ -13191,20 +13170,20 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver // 3.3.1 Get UDAF Evaluator Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, isDistinct); - GenericUDAFEvaluator genericUDAFEvaluator = null; - if (aggName.toLowerCase().equals(FunctionRegistry.LEAD_FUNC_NAME) - || aggName.toLowerCase().equals(FunctionRegistry.LAG_FUNC_NAME)) { - ArrayList originalParameterTypeInfos = + GenericUDAFEvaluator genericUDAFEvaluator; + if (FunctionRegistry.LEAD_FUNC_NAME.equalsIgnoreCase(aggName) + || FunctionRegistry.LAG_FUNC_NAME.equalsIgnoreCase(aggName)) { + List originalParameterTypeInfos = getWritableObjectInspector(aggParameters); genericUDAFEvaluator = FunctionRegistry.getGenericWindowingEvaluator(aggName, originalParameterTypeInfos, isDistinct, isAllColumns); GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); - udafRetType = ((ListTypeInfo)udaf.returnType).getListElementTypeInfo(); + udafRetType = ((ListTypeInfo) udaf.returnType).getListElementTypeInfo(); } else { genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, aggParameters, aggAst, isDistinct, isAllColumns); - assert (genericUDAFEvaluator != null); + assert genericUDAFEvaluator != null; // 3.3.2 Get UDAF Info using UDAF Evaluator GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); @@ -13227,21 +13206,16 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver } // 4. Construct AggInfo - aInfo = new AggInfo(aggParameters, udafRetType, aggName, isDistinct); - return aInfo; + return new AggInfo(aggParameters, udafRetType, aggName, isDistinct); } /** * Generate GB plan. * - * @param qb - * @param srcRel * @return TODO: 1. Grouping Sets (roll up..) - * @throws SemanticException */ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { - RelNode gbRel = null; QBParseInfo qbp = getQBParseInfo(qb); // 0. for GSets, Cube, Rollup, bail from Optiq path. @@ -13249,10 +13223,10 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty()) { String gbyClause = null; - HashMap gbysMap = qbp.getDestToGroupBy(); + Map gbysMap = qbp.getDestToGroupBy(); if (gbysMap.size() == 1) { ASTNode gbyAST = gbysMap.entrySet().iterator().next().getValue(); - gbyClause = SemanticAnalyzer.this.ctx.getTokenRewriteStream() + gbyClause = ctx.getTokenRewriteStream() .toString(gbyAST.getTokenStartIndex(), gbyAST.getTokenStopIndex()); gbyClause = "in '" + gbyClause + "'."; @@ -13271,39 +13245,40 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // NOTE: Multi Insert is not supported String detsClauseName = qbp.getClauseNames().iterator().next(); List grpByAstExprs = getGroupByForClause(qbp, detsClauseName); - HashMap aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); - boolean hasGrpByAstExprs = (grpByAstExprs != null && !grpByAstExprs.isEmpty()) ? true : false; - boolean hasAggregationTrees = (aggregationTrees != null && !aggregationTrees.isEmpty()) ? true - : false; + Map aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); + boolean hasGrpByAstExprs = grpByAstExprs != null && !grpByAstExprs.isEmpty(); + boolean hasAggregationTrees = aggregationTrees != null && !aggregationTrees.isEmpty(); + RelNode gbRel = null; if (hasGrpByAstExprs || hasAggregationTrees) { - ArrayList gbExprNDescLst = new ArrayList(); - ArrayList outputColumnNames = new ArrayList(); + List gbExprNDescLst = new ArrayList(); + List outputColumnNames = new ArrayList(); // 2. Input, Output Row Resolvers - RowResolver groupByInputRowResolver = this.relToHiveRR.get(srcRel); + RowResolver groupByInputRowResolver = relToHiveRR.get(srcRel); RowResolver groupByOutputRowResolver = new RowResolver(); groupByOutputRowResolver.setIsExprResolver(true); if (hasGrpByAstExprs) { // 3. Construct GB Keys (ExprNode) - for (int i = 0; i < grpByAstExprs.size(); ++i) { - ASTNode grpbyExpr = grpByAstExprs.get(i); - Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); + for (ASTNode grpbyExpr : grpByAstExprs) { + Map astToExprNDescMap = + TypeCheckProcFactory.genExprNode(grpbyExpr, + new TypeCheckCtx(groupByInputRowResolver)); ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); - if (grpbyExprNDesc == null) + if (grpbyExprNDesc == null) { throw new RuntimeException("Invalid Column Reference: " + grpbyExpr.dump()); + } addToGBExpr(groupByOutputRowResolver, groupByInputRowResolver, grpbyExpr, - grpbyExprNDesc, gbExprNDescLst, outputColumnNames); + grpbyExprNDesc, gbExprNDescLst, outputColumnNames); } } // 4. Construct aggregation function Info - ArrayList aggregations = new ArrayList(); + List aggregations = new ArrayList(); if (hasAggregationTrees) { - assert (aggregationTrees != null); + assert aggregationTrees != null; for (ASTNode value : aggregationTrees.values()) { // 4.1 Determine type of UDAF // This is the GenericUDAF name @@ -13312,7 +13287,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR; // 4.2 Convert UDAF Params to ExprNodeDesc - ArrayList aggParameters = new ArrayList(); + List aggParameters = new ArrayList(); for (int i = 1; i < value.getChildCount(); i++) { ASTNode paraExpr = (ASTNode) value.getChild(i); ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, groupByInputRowResolver); @@ -13322,13 +13297,13 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, aggParameters, value, isDistinct, isAllColumns); - assert (genericUDAFEvaluator != null); + assert genericUDAFEvaluator != null; GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); AggInfo aInfo = new AggInfo(aggParameters, udaf.returnType, aggName, isDistinct); aggregations.add(aInfo); String field = getColumnInternalName(gbExprNDescLst.size() + aggregations.size() - 1); outputColumnNames.add(field); - groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.m_returnType, + groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.returnType, "", false)); } } @@ -13336,7 +13311,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException gbRel = genGBRelNode(gbExprNDescLst, aggregations, srcRel); relToHiveColNameOptiqPosMap.put(gbRel, buildHiveToOptiqColumnMap(groupByOutputRowResolver, gbRel)); - this.relToHiveRR.put(gbRel, groupByOutputRowResolver); + relToHiveRR.put(gbRel, groupByOutputRowResolver); } return gbRel; @@ -13347,9 +13322,6 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException * introduce top constraining Project. If Input select RelNode is not * present then don't introduce top constraining select. * - * @param qb - * @param srcRel - * @param outermostOB * @return Pair Key- OB RelNode, Value - Input Select for * top constraining Select * @throws SemanticException @@ -13367,7 +13339,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 1. OB Expr sanity test // in strict mode, in the presence of order by, limit must be specified Integer limit = qb.getParseInfo().getDestLimit(dest); - if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") + if ("strict".equalsIgnoreCase(conf.getVar(ConfVars.HIVEMAPREDMODE)) && limit == null) { throw new SemanticException(generateErrorMessage(obAST, ErrorMsg.NO_LIMIT_WITH_ORDERBY.getMsg())); @@ -13377,45 +13349,44 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // virtual columns needed final List newVCLst = new ArrayList(); final List fieldCollations = Lists.newArrayList(); - int fieldIndex = 0; List obASTExprLst = obAST.getChildren(); - ASTNode obASTExpr; List> vcASTTypePairs = new ArrayList>(); RowResolver inputRR = relToHiveRR.get(srcRel); RowResolver outputRR = new RowResolver(); - RexNode rnd; RexNodeConverter converter = new RexNodeConverter(cluster, srcRel.getRowType(), relToHiveColNameOptiqPosMap.get(srcRel), 0, false); int srcRelRecordSz = srcRel.getRowType().getFieldCount(); - for (int i = 0; i < obASTExprLst.size(); i++) { + for (Node anObASTExprLst : obASTExprLst) { // 2.1 Convert AST Expr to ExprNode - obASTExpr = (ASTNode) obASTExprLst.get(i); - Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - obASTExpr, new TypeCheckCtx(inputRR)); + ASTNode obASTExpr = (ASTNode) anObASTExprLst; + Map astToExprNDescMap = + TypeCheckProcFactory.genExprNode(obASTExpr, new TypeCheckCtx(inputRR)); ExprNodeDesc obExprNDesc = astToExprNDescMap.get(obASTExpr.getChild(0)); - if (obExprNDesc == null) - throw new SemanticException("Invalid order by expression: " + obASTExpr.toString()); + if (obExprNDesc == null) { + throw new SemanticException("Invalid order by expression: " + obASTExpr); + } // 2.2 Convert ExprNode to RexNode - rnd = converter.convert(obExprNDesc); + RexNode rnd = converter.convert(obExprNDesc); // 2.3 Determine the index of ob expr in child schema // NOTE: Optiq can not take compound exprs in OB without it being // present in the child (& hence we add a child Project Rel) + int fieldIndex; if (rnd instanceof RexInputRef) { fieldIndex = ((RexInputRef) rnd).getIndex(); } else { fieldIndex = srcRelRecordSz + newVCLst.size(); newVCLst.add(rnd); vcASTTypePairs.add(new Pair((ASTNode) obASTExpr.getChild(0), - obExprNDesc.getTypeInfo())); + obExprNDesc.getTypeInfo())); } // 2.4 Determine the Direction of order by - org.eigenbase.rel.RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING; + RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING; if (obASTExpr.getType() == HiveParser.TOK_TABSORTCOLNAMEASC) { order = RelFieldCollation.Direction.ASCENDING; } @@ -13427,24 +13398,27 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 3. Add Child Project Rel if needed, Generate Output RR, input Sel Rel // for top constraining Sel RelNode obInputRel = srcRel; - if (!newVCLst.isEmpty()) { + if (newVCLst.isEmpty()) { + RowResolver.add(outputRR, inputRR, 0); + } else { List originalInputRefs = Lists.transform(srcRel.getRowType().getFieldList(), - new Function() { - @Override - public RexNode apply(RelDataTypeField input) { - return new RexInputRef(input.getIndex(), input.getType()); - } - }); + new Function() { + @Override + public RexNode apply(RelDataTypeField input) { + return new RexInputRef(input.getIndex(), input.getType()); + } + }); RowResolver obSyntheticProjectRR = new RowResolver(); RowResolver.add(obSyntheticProjectRR, inputRR, 0); int vcolPos = inputRR.getRowSchema().getSignature().size(); for (Pair astTypePair : vcASTTypePairs) { - obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo( - getColumnInternalName(vcolPos), astTypePair.getValue(), null, false)); + obSyntheticProjectRR.putExpression(astTypePair.getKey(), + new ColumnInfo(getColumnInternalName(vcolPos), astTypePair.getValue(), null, false)); vcolPos++; } - obInputRel = genSelectRelNode(CompositeList.of(originalInputRefs, newVCLst), - obSyntheticProjectRR, srcRel); + obInputRel = + genSelectRelNode(CompositeList.of(originalInputRefs, newVCLst), obSyntheticProjectRR, + srcRel); if (outermostOB) { RowResolver.add(outputRR, inputRR, 0); @@ -13453,8 +13427,6 @@ public RexNode apply(RelDataTypeField input) { RowResolver.add(outputRR, obSyntheticProjectRR, 0); originalOBChild = srcRel; } - } else { - RowResolver.add(outputRR, inputRR, 0); } // 4. Construct SortRel @@ -13474,7 +13446,7 @@ public RexNode apply(RelDataTypeField input) { relToHiveColNameOptiqPosMap.put(sortRel, hiveColNameOptiqPosMap); } - return (new Pair(sortRel, originalOBChild)); + return new Pair(sortRel, originalOBChild); } private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { @@ -13499,8 +13471,8 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept return sortRel; } - List getPartitionKeys(PartitionSpec ps, RexNodeConverter converter, RowResolver inputRR) - throws SemanticException { + List getPartitionKeys(PartitionSpec ps, RexNodeConverter converter, + RowResolver inputRR) throws SemanticException { List pKeys = new ArrayList(); if (ps != null) { List pExprs = ps.getExpressions(); @@ -13526,8 +13498,9 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept ExprNodeDesc exp = genExprNodeDesc(oExpr.getExpression(), inputRR, tcCtx); RexNode ordExp = converter.convert(exp); Set flags = new HashSet(); - if (oExpr.getOrder() == org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.DESC) + if (oExpr.getOrder() == PTFInvocationSpec.Order.DESC) { flags.add(SqlKind.DESCENDING); + } oKeys.add(new RexFieldCollation(ordExp, flags)); } } @@ -13539,18 +13512,18 @@ RexWindowBound getBound(BoundarySpec bs, RexNodeConverter converter) { RexWindowBound rwb = null; if (bs != null) { - SqlNode sn = null; SqlParserPos pos = new SqlParserPos(1, 1); SqlNode amt = bs.getAmt() == 0 ? null : SqlLiteral.createExactNumeric( String.valueOf(bs.getAmt()), new SqlParserPos(2, 2)); RexNode amtLiteral = null; - SqlCall sc = null; - RexNode rn = null; - if (amt != null) - amtLiteral = cluster.getRexBuilder().makeLiteral(new Integer(bs.getAmt()), - cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), true); + if (amt != null) { + amtLiteral = cluster.getRexBuilder() + .makeLiteral(bs.getAmt(), cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), + true); + } + SqlCall sc; switch (bs.getDirection()) { case PRECEDING: if (amt == null) { @@ -13583,7 +13556,6 @@ RexWindowBound getBound(BoundarySpec bs, RexNodeConverter converter) { } int getWindowSpecIndx(ASTNode wndAST) { - int wndASTIndx = -1; int wi = wndAST.getChildCount() - 1; if (wi <= 0 || (wndAST.getChild(wi).getType() != HiveParser.TOK_WINDOWSPEC)) { wi = -1; @@ -13594,8 +13566,8 @@ int getWindowSpecIndx(ASTNode wndAST) { Pair genWindowingProj(QB qb, WindowExpressionSpec wExpSpec, RelNode srcRel) throws SemanticException { - RexNode w = null; - TypeInfo wHiveRetType = null; + RexNode w; + TypeInfo wHiveRetType; if (wExpSpec instanceof WindowFunctionSpec) { WindowFunctionSpec wFnSpec = (WindowFunctionSpec) wExpSpec; @@ -13604,30 +13576,29 @@ int getWindowSpecIndx(ASTNode wndAST) { int wndSpecASTIndx = getWindowSpecIndx(windowProjAst); // 2. Get Hive Aggregate Info AggInfo hiveAggInfo = getHiveAggInfo(windowProjAst, wndSpecASTIndx - 1, - this.relToHiveRR.get(srcRel)); + relToHiveRR.get(srcRel)); // 3. Get Optiq Return type for Agg Fn - wHiveRetType = hiveAggInfo.m_returnType; - RelDataType optiqAggFnRetType = TypeConverter.convert(hiveAggInfo.m_returnType, - this.cluster.getTypeFactory()); + wHiveRetType = hiveAggInfo.returnType; + RelDataType optiqAggFnRetType = TypeConverter.convert(hiveAggInfo.returnType, + cluster.getTypeFactory()); // 4. Convert Agg Fn args to Optiq - ImmutableMap posMap = this.relToHiveColNameOptiqPosMap.get(srcRel); - RexNodeConverter converter = new RexNodeConverter(this.cluster, srcRel.getRowType(), + ImmutableMap posMap = relToHiveColNameOptiqPosMap.get(srcRel); + RexNodeConverter converter = new RexNodeConverter(cluster, srcRel.getRowType(), posMap, 0, false); - Builder optiqAggFnArgsBldr = ImmutableList. builder(); - Builder optiqAggFnArgsTypeBldr = ImmutableList. builder(); - RexNode rexNd = null; - for (int i = 0; i < hiveAggInfo.m_aggParams.size(); i++) { - optiqAggFnArgsBldr.add(converter.convert(hiveAggInfo.m_aggParams.get(i))); - optiqAggFnArgsTypeBldr.add(TypeConverter.convert(hiveAggInfo.m_aggParams.get(i) - .getTypeInfo(), this.cluster.getTypeFactory())); + Builder optiqAggFnArgsBldr = ImmutableList.builder(); + Builder optiqAggFnArgsTypeBldr = ImmutableList.builder(); + for (int i = 0; i < hiveAggInfo.aggParams.size(); i++) { + optiqAggFnArgsBldr.add(converter.convert(hiveAggInfo.aggParams.get(i))); + optiqAggFnArgsTypeBldr.add(TypeConverter.convert(hiveAggInfo.aggParams.get(i) + .getTypeInfo(), cluster.getTypeFactory())); } ImmutableList optiqAggFnArgs = optiqAggFnArgsBldr.build(); ImmutableList optiqAggFnArgsType = optiqAggFnArgsTypeBldr.build(); // 5. Get Optiq Agg Fn - final SqlAggFunction optiqAggFn = SqlFunctionConverter.getOptiqAggFn(hiveAggInfo.m_udfName, + final SqlAggFunction optiqAggFn = SqlFunctionConverter.getOptiqAggFn(hiveAggInfo.udfName, optiqAggFnArgsType, optiqAggFnRetType); // 6. Translate Window spec @@ -13637,25 +13608,25 @@ int getWindowSpecIndx(ASTNode wndAST) { List orderKeys = getOrderKeys(wndSpec.getOrder(), converter, inputRR); RexWindowBound upperBound = getBound(wndSpec.windowFrame.start, converter); RexWindowBound lowerBound = getBound(wndSpec.windowFrame.end, converter); - boolean isRows = ((wndSpec.windowFrame.start instanceof RangeBoundarySpec) || (wndSpec.windowFrame.end instanceof RangeBoundarySpec)) ? true - : false; + boolean isRows = (wndSpec.windowFrame.start instanceof RangeBoundarySpec) + || (wndSpec.windowFrame.end instanceof RangeBoundarySpec); w = cluster.getRexBuilder().makeOver(optiqAggFnRetType, optiqAggFn, optiqAggFnArgs, - partitionKeys, ImmutableList. copyOf(orderKeys), lowerBound, + partitionKeys, ImmutableList.copyOf(orderKeys), lowerBound, upperBound, isRows, true, false); } else { // TODO: Convert to Semantic Exception throw new RuntimeException("Unsupported window Spec"); } - return new Pair(w, wHiveRetType); + return new Pair(w, wHiveRetType); } private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticException { RelNode selOpForWindow = null; - QBParseInfo qbp = getQBParseInfo(qb); - WindowingSpec wSpec = (!qb.getAllWindowingSpecs().isEmpty()) ? qb.getAllWindowingSpecs() - .values().iterator().next() : null; + WindowingSpec wSpec = qb.getAllWindowingSpecs().isEmpty() + ? null + : qb.getAllWindowingSpecs().values().iterator().next(); if (wSpec != null) { // 1. Get valid Window Function Spec @@ -13663,58 +13634,58 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticExce List windowExpressions = wSpec.getWindowExpressions(); if (windowExpressions != null && !windowExpressions.isEmpty()) { - RowResolver inputRR = this.relToHiveRR.get(srcRel); + RowResolver inputRR = relToHiveRR.get(srcRel); // 2. Get RexNodes for original Projections from below List projsForWindowSelOp = new ArrayList( HiveOptiqUtil.getProjsFromBelowAsInputRef(srcRel)); // 3. Construct new Row Resolver with everything from below. - RowResolver out_rwsch = new RowResolver(); - RowResolver.add(out_rwsch, inputRR, 0); + RowResolver outRwsch = new RowResolver(); + RowResolver.add(outRwsch, inputRR, 0); // 4. Walk through Window Expressions & Construct RexNodes for those, // Update out_rwsch for (WindowExpressionSpec wExprSpec : windowExpressions) { - if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) { + if (outRwsch.getExpression(wExprSpec.getExpression()) == null) { Pair wtp = genWindowingProj(qb, wExprSpec, srcRel); projsForWindowSelOp.add(wtp.getKey()); // 6.2.2 Update Output Row Schema ColumnInfo oColInfo = new ColumnInfo( getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); + outRwsch.putExpression(wExprSpec.getExpression(), oColInfo); + /* + TODO: Not sure what this code does or if it can be removed if (false) { - out_rwsch.checkColumn(null, wExprSpec.getAlias()); - out_rwsch.put(null, wExprSpec.getAlias(), oColInfo); - } else { - out_rwsch.putExpression(wExprSpec.getExpression(), oColInfo); + outRwsch.checkColumn(null, wExprSpec.getAlias()); + outRwsch.put(null, wExprSpec.getAlias(), oColInfo); } + */ } } - selOpForWindow = genSelectRelNode(projsForWindowSelOp, out_rwsch, srcRel); + selOpForWindow = genSelectRelNode(projsForWindowSelOp, outRwsch, srcRel); } } return selOpForWindow; } - private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsch, + private RelNode genSelectRelNode(List optiqColLst, RowResolver outRwsch, RelNode srcRel) throws OptiqSemanticException { // 1. Build Column Names Set colNamesSet = new HashSet(); - List cInfoLst = out_rwsch.getRowSchema().getSignature(); - ArrayList columnNames = new ArrayList(); - String[] qualifiedColNames; - String tmpColAlias; + List cInfoLst = outRwsch.getRowSchema().getSignature(); + List columnNames = new ArrayList(); for (int i = 0; i < optiqColLst.size(); i++) { ColumnInfo cInfo = cInfoLst.get(i); - qualifiedColNames = out_rwsch.reverseLookup(cInfo.getInternalName()); + String[] qualifiedColNames = outRwsch.reverseLookup(cInfo.getInternalName()); /* if (qualifiedColNames[0] != null && !qualifiedColNames[0].isEmpty()) tmpColAlias = qualifiedColNames[0] + "." + qualifiedColNames[1]; else */ - tmpColAlias = qualifiedColNames[1]; + String tmpColAlias = qualifiedColNames[1]; // Prepend column names with '_o_' if it starts with '_c' /* @@ -13722,8 +13693,9 @@ private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsc * the names so we don't run into this issue when converting back to * Hive AST. */ - if (tmpColAlias.startsWith("_c")) + if (tmpColAlias.startsWith("_c")) { tmpColAlias = "_o_" + tmpColAlias; + } int suffix = 1; while (colNamesSet.contains(tmpColAlias)) { tmpColAlias = qualifiedColNames[1] + suffix; @@ -13739,17 +13711,17 @@ private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsc HiveRel selRel = HiveProjectRel.create(srcRel, optiqColLst, columnNames); // 4. Keep track of colname-to-posmap && RR for new select - this.relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(out_rwsch, selRel)); - this.relToHiveRR.put(selRel, out_rwsch); + relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(outRwsch, selRel)); + relToHiveRR.put(selRel, outRwsch); return selRel; } - private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsch, + private RelNode genSelectRelNode(List optiqColLst, RowResolver outRwsch, RelNode srcRel, boolean removethismethod) throws OptiqSemanticException { // 1. Build Column Names // TODO: Should this be external names - ArrayList columnNames = new ArrayList(); + List columnNames = new ArrayList(); for (int i = 0; i < optiqColLst.size(); i++) { columnNames.add(getColumnInternalName(i)); } @@ -13771,8 +13743,8 @@ public String apply(String hName) { HiveRel selRel = HiveProjectRel.create(srcRel, optiqColLst, oFieldNames); // 4. Keep track of colname-to-posmap && RR for new select - this.relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(out_rwsch, selRel)); - this.relToHiveRR.put(selRel, out_rwsch); + relToHiveColNameOptiqPosMap.put(selRel, buildHiveToOptiqColumnMap(outRwsch, selRel)); + relToHiveRR.put(selRel, outRwsch); return selRel; } @@ -13789,9 +13761,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep RelNode selForWindow = genSelectForWindowing(qb, srcRel); srcRel = (selForWindow == null) ? srcRel : selForWindow; - boolean subQuery; - ArrayList col_list = new ArrayList(); - ArrayList> windowingRexNodes = new ArrayList>(); + List colList = new ArrayList(); // 1. Get Select Expression List QBParseInfo qbp = getQBParseInfo(qb); @@ -13799,18 +13769,14 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep ASTNode selExprList = qbp.getSelForClause(selClauseName); // 2.Row resolvers for input, output - RowResolver out_rwsch = new RowResolver(); - ASTNode trfm = null; - Integer pos = Integer.valueOf(0); - RowResolver inputRR = this.relToHiveRR.get(srcRel); + RowResolver outRwsch = new RowResolver(); + RowResolver inputRR = relToHiveRR.get(srcRel); // 3. Query Hints // TODO: Handle Query Hints; currently we ignore them - boolean selectStar = false; - int posn = 0; - boolean hintPresent = (selExprList.getChild(0).getType() == HiveParser.TOK_HINTLIST); + boolean hintPresent = selExprList.getChild(0).getType() == HiveParser.TOK_HINTLIST; if (hintPresent) { - String hint = SemanticAnalyzer.this.ctx.getTokenRewriteStream(). + String hint = ctx.getTokenRewriteStream(). toString( selExprList.getChild(0).getTokenStartIndex(), selExprList.getChild(0).getTokenStopIndex()); @@ -13821,45 +13787,46 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep } // 4. Determine if select corresponds to a subquery - subQuery = qb.getParseInfo().getIsSubQ(); + boolean subQuery = qb.getParseInfo().getIsSubQ(); // 4. Bailout if select involves Transform - boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); + int posn = 0; + boolean isInTransform = + selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM; if (isInTransform) { - String msg = String.format("SELECT TRANSFORM is currently not supported in CBO," - + " turn off cbo to use TRANSFORM."); + String msg = "SELECT TRANSFORM is currently not supported in CBO," + + " turn off cbo to use TRANSFORM."; LOG.debug(msg); throw new OptiqSemanticException(msg); } // 5. Bailout if select involves UDTF ASTNode udtfExpr = (ASTNode) selExprList.getChild(posn).getChild(0); - GenericUDTF genericUDTF = null; int udtfExprType = udtfExpr.getType(); if (udtfExprType == HiveParser.TOK_FUNCTION || udtfExprType == HiveParser.TOK_FUNCTIONSTAR) { String funcName = TypeCheckProcFactory.DefaultExprProcessor.getFunctionText(udtfExpr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); + GenericUDTF genericUDTF = null; if (fi != null) { genericUDTF = fi.getGenericUDTF(); } if (genericUDTF != null) { - String msg = String.format("UDTF " + funcName + " is currently not supported in CBO," - + " turn off cbo to use UDTF " + funcName); + String msg = "UDTF " + funcName + " is currently not supported in CBO," + + " turn off cbo to use UDTF " + funcName; LOG.debug(msg); throw new OptiqSemanticException(msg); } } // 6. Iterate over all expression (after SELECT) - ASTNode exprList = selExprList; - int startPosn = posn; - int wndProjPos = 0; List tabAliasesForAllProjs = getTabAliases(inputRR); - for (int i = startPosn; i < exprList.getChildCount(); ++i) { + boolean selectStar = false; + Integer pos = 0; + for (int i = posn; i < selExprList.getChildCount(); ++i) { // 6.1 child can be EXPR AS ALIAS, or EXPR. - ASTNode child = (ASTNode) exprList.getChild(i); - boolean hasAsClause = (!isInTransform) && (child.getChildCount() == 2); + ASTNode child = (ASTNode) selExprList.getChild(i); + boolean hasAsClause = !isInTransform && (child.getChildCount() == 2); // 6.2 EXPR AS (ALIAS,...) parses, but is only allowed for UDTF's // This check is not needed and invalid when there is a transform b/c @@ -13870,22 +13837,18 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep ErrorMsg.INVALID_AS.getMsg())); } - ASTNode expr; - String tabAlias; - String colAlias; - // 6.3 Get rid of TOK_SELEXPR - expr = (ASTNode) child.getChild(0); + ASTNode expr = (ASTNode) child.getChild(0); String[] colRef = getColAlias(child, autogenColAliasPrfxLbl, inputRR, autogenColAliasPrfxIncludeFuncName, i); - tabAlias = colRef[0]; - colAlias = colRef[1]; + String tabAlias = colRef[0]; + String colAlias = colRef[1]; // 6.4 Build ExprNode corresponding to colums if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)) - .toLowerCase(), expr, col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, + .toLowerCase(), expr, colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, subQuery); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause @@ -13895,7 +13858,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, subQuery); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText() @@ -13906,7 +13869,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + colList, inputRR, pos, outRwsch, tabAliasesForAllProjs, subQuery); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -13914,45 +13877,44 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep tcCtx.setAllowStatefulFunctions(true); ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx); String recommended = recommendName(exp, colAlias); - if (recommended != null && out_rwsch.get(null, recommended) == null) { + if (recommended != null && outRwsch.get(null, recommended) == null) { colAlias = recommended; } - col_list.add(exp); + colList.add(exp); if (subQuery) { - out_rwsch.checkColumn(tabAlias, colAlias); + outRwsch.checkColumn(tabAlias, colAlias); } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); - colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) ? ((ExprNodeColumnDesc) exp) - .isSkewedCol() : false); - out_rwsch.put(tabAlias, colAlias, colInfo); + colInfo.setSkewedCol( + (exp instanceof ExprNodeColumnDesc) && ((ExprNodeColumnDesc) exp).isSkewedCol()); + outRwsch.put(tabAlias, colAlias, colInfo); if (exp instanceof ExprNodeColumnDesc) { ExprNodeColumnDesc colExp = (ExprNodeColumnDesc) exp; String[] altMapping = inputRR.getAlternateMappings(colExp.getColumn()); if (altMapping != null) { - out_rwsch.put(altMapping[0], altMapping[1], colInfo); + outRwsch.put(altMapping[0], altMapping[1], colInfo); } } - pos = Integer.valueOf(pos.intValue() + 1); + pos += 1; } } - selectStar = selectStar && exprList.getChildCount() == posn + 1; + selectStar = selectStar && selExprList.getChildCount() == posn + 1; // 7. Convert Hive projections to Optiq List optiqColLst = new ArrayList(); RexNodeConverter rexNodeConv = new RexNodeConverter(cluster, srcRel.getRowType(), - buildHiveColNameToInputPosMap(col_list, inputRR), 0, false); - for (ExprNodeDesc colExpr : col_list) { + buildHiveColNameToInputPosMap(colList, inputRR), 0, false); + for (ExprNodeDesc colExpr : colList) { optiqColLst.add(rexNodeConv.convert(colExpr)); } // 8. Build Optiq Rel - RelNode selRel = genSelectRelNode(optiqColLst, out_rwsch, srcRel); - return selRel; + return genSelectRelNode(optiqColLst, outRwsch, srcRel); } private RelNode genLogicalPlan(QBExpr qbexpr) throws SemanticException { @@ -13970,23 +13932,13 @@ private RelNode genLogicalPlan(QBExpr qbexpr) throws SemanticException { } private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticException { - RelNode srcRel = null; - RelNode filterRel = null; - RelNode gbRel = null; - RelNode gbHavingRel = null; - RelNode havingRel = null; - RelNode selectRel = null; - RelNode obRel = null; - RelNode limitRel = null; - - RelNode rootRel = null; // First generate all the opInfos for the elements in the from clause Map aliasToRel = new HashMap(); // 0. Check if we can handle the query // This check is needed here because of SubQuery if (!canHandleQuery(qb, false)) { - String msg = String.format("CBO Can not handle Sub Query"); + String msg = "CBO Can not handle Sub Query"; LOG.debug(msg); throw new OptiqSemanticException(msg); } @@ -14022,37 +13974,38 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticExcept } // 1.3 process join - if (qb.getParseInfo().getJoinExpr() != null) { - srcRel = genJoinLogicalPlan(qb.getParseInfo().getJoinExpr(), aliasToRel); - } else { + RelNode srcRel = null; + if (qb.getParseInfo().getJoinExpr() == null) { // If no join then there should only be either 1 TS or 1 SubQuery srcRel = aliasToRel.values().iterator().next(); + } else { + srcRel = genJoinLogicalPlan(qb.getParseInfo().getJoinExpr(), aliasToRel); } // 2. Build Rel for where Clause - filterRel = genFilterLogicalPlan(qb, srcRel, aliasToRel, false); + RelNode filterRel = genFilterLogicalPlan(qb, srcRel, aliasToRel, false); srcRel = (filterRel == null) ? srcRel : filterRel; // 3. Build Rel for GB Clause - gbRel = genGBLogicalPlan(qb, srcRel); + RelNode gbRel = genGBLogicalPlan(qb, srcRel); srcRel = (gbRel == null) ? srcRel : gbRel; // 4. Build Rel for GB Having Clause - gbHavingRel = genGBHavingLogicalPlan(qb, srcRel, aliasToRel); + RelNode gbHavingRel = genGBHavingLogicalPlan(qb, srcRel, aliasToRel); srcRel = (gbHavingRel == null) ? srcRel : gbHavingRel; // 5. Build Rel for Select Clause - selectRel = genSelectLogicalPlan(qb, srcRel); + RelNode selectRel = genSelectLogicalPlan(qb, srcRel); srcRel = (selectRel == null) ? srcRel : selectRel; // 6. Build Rel for OB Clause Pair obTopProjPair = genOBLogicalPlan(qb, srcRel, outerMostQB); - obRel = obTopProjPair.getKey(); + RelNode obRel = obTopProjPair.getKey(); RelNode topConstrainingProjArgsRel = obTopProjPair.getValue(); srcRel = (obRel == null) ? srcRel : obRel; // 7. Build Rel for Limit Clause - limitRel = genLimitLogicalPlan(qb, srcRel); + RelNode limitRel = genLimitLogicalPlan(qb, srcRel); srcRel = (limitRel == null) ? srcRel : limitRel; // 8. Introduce top constraining select if needed. @@ -14079,7 +14032,7 @@ public RexNode apply(RelDataTypeField input) { } }); RowResolver topConstrainingProjRR = new RowResolver(); - RowResolver.add(topConstrainingProjRR, this.relToHiveRR.get(topConstrainingProjArgsRel), 0); + RowResolver.add(topConstrainingProjRR, relToHiveRR.get(topConstrainingProjArgsRel), 0); srcRel = genSelectRelNode(originalInputRefs, topConstrainingProjRR, srcRel); } @@ -14087,13 +14040,13 @@ public RexNode apply(RelDataTypeField input) { // to subquery alias // TODO: cleanup this if (qb.getParseInfo().getAlias() != null) { - RowResolver rr = this.relToHiveRR.get(srcRel); + RowResolver rr = relToHiveRR.get(srcRel); RowResolver newRR = new RowResolver(); String alias = qb.getParseInfo().getAlias(); for (ColumnInfo colInfo : rr.getColumnInfos()) { String name = colInfo.getInternalName(); String[] tmp = rr.reverseLookup(name); - if ("".equals(tmp[0]) || tmp[1] == null) { + if (tmp[0] != null && tmp[0].isEmpty() || tmp[1] == null) { // ast expression is not a valid column name for table tmp[1] = colInfo.getInternalName(); } @@ -14175,8 +14128,8 @@ public Object post(Object t) { }; new TreeVisitor(ParseDriver.adaptor).visit(havingExpr, action); - if (aliasReferences.size() > 0) { - String havingClause = SemanticAnalyzer.this.ctx + if (!aliasReferences.isEmpty()) { + String havingClause = ctx .getTokenRewriteStream().toString( havingExpr.getTokenStartIndex(), havingExpr.getTokenStopIndex()); @@ -14194,7 +14147,6 @@ public Object post(Object t) { private ImmutableMap buildHiveToOptiqColumnMap(RowResolver rr, RelNode rNode) { ImmutableMap.Builder b = new ImmutableMap.Builder(); - int i = 0; for (ColumnInfo ci : rr.getRowSchema().getSignature()) { b.put(ci.getInternalName(), rr.getPosition(ci.getInternalName())); } @@ -14202,15 +14154,15 @@ public Object post(Object t) { } private ImmutableMap buildHiveColNameToInputPosMap( - List col_list, RowResolver inputRR) { + List colList, RowResolver inputRR) { // Build a map of Hive column Names (ExprNodeColumnDesc Name) // to the positions of those projections in the input Map hashCodeTocolumnDescMap = new HashMap(); - ExprNodeDescUtils.getExprNodeColumnDesc(col_list, hashCodeTocolumnDescMap); - ImmutableMap.Builder hiveColNameToInputPosMapBuilder = new ImmutableMap.Builder(); - String exprNodecolName; + ExprNodeDescUtils.getExprNodeColumnDesc(colList, hashCodeTocolumnDescMap); + ImmutableMap.Builder hiveColNameToInputPosMapBuilder = + new ImmutableMap.Builder(); for (ExprNodeDesc exprDesc : hashCodeTocolumnDescMap.values()) { - exprNodecolName = ((ExprNodeColumnDesc) exprDesc).getColumn(); + String exprNodecolName = ((ExprNodeColumnDesc) exprDesc).getColumn(); hiveColNameToInputPosMapBuilder.put(exprNodecolName, inputRR.getPosition(exprNodecolName)); } @@ -14220,8 +14172,8 @@ public Object post(Object t) { private QBParseInfo getQBParseInfo(QB qb) throws OptiqSemanticException { QBParseInfo qbp = qb.getParseInfo(); if (qbp.getClauseNames().size() > 1) { - String msg = String.format("Multi Insert is currently not supported in CBO," - + " turn off cbo to use Multi Insert."); + String msg = "Multi Insert is currently not supported in CBO," + + " turn off cbo to use Multi Insert."; LOG.debug(msg); throw new OptiqSemanticException(msg); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index e065983..ecdd9f3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -118,7 +118,7 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) RowResolver input = ctx.getInputRR(); ExprNodeDesc desc = null; - if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) { + if ((ctx == null) || (input == null) || !ctx.getAllowGBExprElimination()) { return null; } @@ -721,10 +721,10 @@ protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, F throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr)); } } - if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) { - if (FunctionRegistry.isStateful(genericUDF)) { - throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg()); - } + if (!ctx.getAllowStatefulFunctions() + && (genericUDF != null) + && FunctionRegistry.isStateful(genericUDF)) { + throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg()); } } @@ -1029,9 +1029,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, * return null; */ if (windowingTokens.contains(expr.getType())) { - if (!ctx.getallowWindowing()) + if (!ctx.getallowWindowing()) { throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); + ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); + } return null; } @@ -1041,10 +1042,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (expr.getType() == HiveParser.TOK_ALLCOLREF) { - if (!ctx.getallowAllColRef()) + if (!ctx.getallowAllColRef()) { throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN - .getMsg("All column reference is not supported in the context"))); + ErrorMsg.INVALID_COLUMN.getMsg( + "All column reference is not supported in the context"))); + } RowResolver input = ctx.getInputRR(); ExprNodeColumnListDesc columnList = new ExprNodeColumnListDesc(); @@ -1117,10 +1119,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) { - if (!ctx.getallowFunctionStar()) - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN - .getMsg(".* reference is not supported in the context"))); + if (!ctx.getallowFunctionStar()) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, + ErrorMsg.INVALID_COLUMN.getMsg(".* reference is not supported in the context"))); + } RowResolver input = ctx.getInputRR(); for (ColumnInfo colInfo : input.getColumnInfos()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java index 1a0cdf8..7a972fa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java @@ -20,8 +20,9 @@ import java.io.Externalizable; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; @@ -31,7 +32,7 @@ * AggregationDesc. * */ -public class AggregationDesc implements java.io.Serializable { +public class AggregationDesc implements Serializable { static { PTFUtils.makeTransient(AggregationDesc.class, "genericUDAFEvaluator"); @@ -40,7 +41,7 @@ private static final long serialVersionUID = 1L; private String genericUDAFName; - private java.util.ArrayList parameters; + private List parameters; private boolean distinct; private GenericUDAFEvaluator.Mode mode; @@ -56,7 +57,7 @@ public AggregationDesc() { public AggregationDesc(final String genericUDAFName, final GenericUDAFEvaluator genericUDAFEvaluator, - final java.util.ArrayList parameters, + final List parameters, final boolean distinct, final GenericUDAFEvaluator.Mode mode) { this.genericUDAFName = genericUDAFName; this.parameters = parameters; @@ -116,11 +117,11 @@ public void setGenericUDAFWritableEvaluator(GenericUDAFEvaluator genericUDAFWrit this.genericUDAFWritableEvaluator = genericUDAFWritableEvaluator; } - public java.util.ArrayList getParameters() { + public List getParameters() { return parameters; } - public void setParameters(final java.util.ArrayList parameters) { + public void setParameters(final ArrayList parameters) { this.parameters = parameters; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index 8a41577..6d55ef8 100755 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -94,8 +94,9 @@ public String getExprString() { return "null"; } - if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) || typeInfo instanceof BaseCharTypeInfo) { - return "'" + value.toString() + "'"; + if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) + || typeInfo instanceof BaseCharTypeInfo) { + return "'" + value + "'"; } else if (typeInfo.getTypeName().equals(serdeConstants.BINARY_TYPE_NAME)) { byte[] bytes = (byte[]) value; char[] hexChars = new char[bytes.length * 2]; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index 686befd..3ba87cd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -383,42 +383,36 @@ private static ExprNodeConstantDesc foldConstant(ExprNodeGenericFuncDesc func) { } catch (Exception e) { return null; } - } + } - public static void getExprNodeColumnDesc(List exprDescList, - Map hashCodeTocolumnDescMap) { - for (ExprNodeDesc exprNodeDesc : exprDescList) { - getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); - } - } + public static void getExprNodeColumnDesc(List exprDescList, + Map hashCodeTocolumnDescMap) { + for (ExprNodeDesc exprNodeDesc : exprDescList) { + getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); + } + } - /** - * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. - * - * @param exprDesc - * @param hashCodeTocolumnDescMap - * Assumption: If two ExprNodeColumnDesc have same hash code then - * they are logically referring to same projection - */ - public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, - Map hashCodeTocolumnDescMap) { - if (exprDesc instanceof ExprNodeColumnDesc) { - hashCodeTocolumnDescMap.put( - ((ExprNodeColumnDesc) exprDesc).hashCode(), - ((ExprNodeColumnDesc) exprDesc)); - } else if (exprDesc instanceof ExprNodeColumnListDesc) { - for (ExprNodeDesc child : ((ExprNodeColumnListDesc) exprDesc) - .getChildren()) { - getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); - } - } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { - for (ExprNodeDesc child : ((ExprNodeGenericFuncDesc) exprDesc) - .getChildren()) { - getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); - } - } else if (exprDesc instanceof ExprNodeFieldDesc) { - getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), - hashCodeTocolumnDescMap); - } - } + /** + * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. + * + * @param hashCodeTocolumnDescMap + * Assumption: If two ExprNodeColumnDesc have same hash code then + * they are logically referring to same projection + */ + public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, + Map hashCodeTocolumnDescMap) { + if (exprDesc instanceof ExprNodeColumnDesc) { + hashCodeTocolumnDescMap.put(exprDesc.hashCode(), exprDesc); + } else if (exprDesc instanceof ExprNodeColumnListDesc) { + for (ExprNodeDesc child : exprDesc.getChildren()) { + getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); + } + } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { + for (ExprNodeDesc child : exprDesc.getChildren()) { + getExprNodeColumnDesc(child, hashCodeTocolumnDescMap); + } + } else if (exprDesc instanceof ExprNodeFieldDesc) { + getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), hashCodeTocolumnDescMap); + } + } }