diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java index bd346eb..1588cdf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java @@ -23,12 +23,6 @@ public HiveAggregateRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode chi BitSet groupSet, List aggCalls) throws InvalidRelException { super(cluster, TraitsUtil.getAggregateTraitSet(cluster, traitSet, BitSets.toList(groupSet), aggCalls, child), child, groupSet, aggCalls); - - for (AggregateCall aggCall : aggCalls) { - if (aggCall.isDistinct()) { - throw new InvalidRelException("distinct aggregation not supported"); - } - } } public AggregateRelBase copy(RelTraitSet traitSet, RelNode input, BitSet groupSet, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java index d0d7715..cb16836 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java @@ -100,7 +100,9 @@ public ASTNode convert(SortRel sortrel) { RexInputRef iRef = new RexInputRef(i, new BasicSqlType(SqlTypeName.ANY)); b.add(iRef.accept(new RexVisitor(schema))); } - hiveAST.groupBy = b.node(); + + if (!groupBy.getGroupSet().isEmpty()) + hiveAST.groupBy = b.node(); schema = new Schema(schema, groupBy); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c516657..4fd3861 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12159,7 +12159,7 @@ private AggInfo(List aggParams, TypeInfo returnType, private AggregateCall convertAgg(AggInfo agg, RelNode input, List gbChildProjLst, RexNodeConverter converter, - HashMap rexNodeToPosMap, Integer childProjLstIndx) + HashMap rexNodeToPosMap, Integer childProjLstIndx) throws SemanticException { final Aggregation aggregation = AGG_MAP.get(agg.m_udfName); if (aggregation == null) { @@ -12176,10 +12176,10 @@ private AggregateCall convertAgg(AggInfo agg, RelNode input, RexNode rexNd = null; for (ExprNodeDesc expr : agg.m_aggParams) { rexNd = converter.convert(expr); - inputIndx = rexNodeToPosMap.get(rexNd); + inputIndx = rexNodeToPosMap.get(rexNd.toString()); if (inputIndx == null) { gbChildProjLst.add(rexNd); - rexNodeToPosMap.put(rexNd, childProjLstIndx); + rexNodeToPosMap.put(rexNd.toString(), childProjLstIndx); inputIndx = childProjLstIndx; childProjLstIndx++; } @@ -12207,7 +12207,7 @@ private RelNode genGBRelNode(List gbExprs, srcRel.getRowType(), posMap, 0, false); final List gbChildProjLst = Lists.newArrayList(); - final HashMap rexNodeToPosMap = new HashMap(); + final HashMap rexNodeToPosMap = new HashMap(); final BitSet groupSet = new BitSet(); Integer gbIndx = 0; RexNode rnd; @@ -12215,7 +12215,7 @@ private RelNode genGBRelNode(List gbExprs, rnd = converter.convert(key); gbChildProjLst.add(rnd); groupSet.set(gbIndx); - rexNodeToPosMap.put(rnd, gbIndx); + rexNodeToPosMap.put(rnd.toString(), gbIndx); gbIndx++; } @@ -12269,6 +12269,24 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, } } + private void addToGBExpr(RowResolver groupByOutputRowResolver, + RowResolver groupByInputRowResolver, ASTNode grpbyExpr, ExprNodeDesc grpbyExprNDesc, List gbExprNDescLst, + List outputColumnNames) { + // TODO: Should we use grpbyExprNDesc.getTypeInfo()? what if expr is + // UDF + int i = gbExprNDescLst.size(); + String field = getColumnInternalName(i); + outputColumnNames.add(field); + gbExprNDescLst.add(grpbyExprNDesc); + + ColumnInfo oColInfo = new ColumnInfo(field, grpbyExprNDesc.getTypeInfo(), null, false); + groupByOutputRowResolver.putExpression(grpbyExpr, oColInfo); + + // TODO: Alternate mappings, are they necessary? + addAlternateGByKeyMappings(grpbyExpr, oColInfo, groupByInputRowResolver, + groupByOutputRowResolver); + } + /** * Generate GB plan. * @@ -12277,89 +12295,79 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, * @return TODO: 1. Grouping Sets (roll up..) * @throws SemanticException */ - private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) - throws SemanticException { + private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { RelNode gbRel = null; QBParseInfo qbp = getQBParseInfo(qb); - // 1. Gather GB Expressions (AST) + // 1. Gather GB Expressions (AST) (GB + Aggregations) // NOTE: Multi Insert is not supported String detsClauseName = qbp.getClauseNames().iterator().next(); List grpByAstExprs = getGroupByForClause(qbp, detsClauseName); + HashMap aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName); + boolean hasGrpByAstExprs = (grpByAstExprs != null && !grpByAstExprs.isEmpty()) ? true : false; + boolean hasAggregationTrees = (aggregationTrees != null && !aggregationTrees.isEmpty()) ? true + : false; + + if (hasGrpByAstExprs || hasAggregationTrees) { + ArrayList gbExprNDescLst = new ArrayList(); + ArrayList outputColumnNames = new ArrayList(); + int numDistinctUDFs = 0; - if (grpByAstExprs != null && !grpByAstExprs.isEmpty()) { // 2. Input, Output Row Resolvers RowResolver groupByInputRowResolver = this.m_relToHiveRR.get(srcRel); RowResolver groupByOutputRowResolver = new RowResolver(); groupByOutputRowResolver.setIsExprResolver(true); - // 3. Construct GB Keys (ExprNode) - ArrayList gbExprNDescLst = new ArrayList(); - ArrayList outputColumnNames = new ArrayList(); - for (int i = 0; i < grpByAstExprs.size(); ++i) { - ASTNode grpbyExpr = grpByAstExprs.get(i); - Map astToExprNDescMap = TypeCheckProcFactory - .genExprNode(grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); - ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); - if (grpbyExprNDesc == null) - throw new RuntimeException("Invalid Column Reference: " - + grpbyExpr.dump()); - gbExprNDescLst.add(grpbyExprNDesc); - - // TODO: Should we use grpbyExprNDesc.getTypeInfo()? what if expr is - // UDF - String field = getColumnInternalName(i); - outputColumnNames.add(field); - ColumnInfo oColInfo = new ColumnInfo(field, - grpbyExprNDesc.getTypeInfo(), null, false); - groupByOutputRowResolver.putExpression(grpbyExpr, oColInfo); - - // TODO: Alternate mappings, are they necessary? - addAlternateGByKeyMappings(grpbyExpr, oColInfo, - groupByInputRowResolver, groupByOutputRowResolver); + if (hasGrpByAstExprs) { + // 3. Construct GB Keys (ExprNode) + for (int i = 0; i < grpByAstExprs.size(); ++i) { + ASTNode grpbyExpr = grpByAstExprs.get(i); + Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( + grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); + ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); + if (grpbyExprNDesc == null) + throw new RuntimeException("Invalid Column Reference: " + grpbyExpr.dump()); + + addToGBExpr(groupByOutputRowResolver, groupByInputRowResolver, grpbyExpr, + grpbyExprNDesc, gbExprNDescLst, outputColumnNames); + } } // 4. Construct aggregation function Info ArrayList aggregations = new ArrayList(); - HashMap aggregationTrees = qbp - .getAggregationExprsForClause(detsClauseName); - assert (aggregationTrees != null); - int numDistinctUDFs = 0; - for (ASTNode value : aggregationTrees.values()) { - - // 4.1 Convert UDF Params to ExprNodeDesc - ArrayList aggParameters = new ArrayList(); - for (int i = 1; i < value.getChildCount(); i++) { - ASTNode paraExpr = (ASTNode) value.getChild(i); - ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, - groupByInputRowResolver); - aggParameters.add(paraExprNode); - } + if (hasAggregationTrees) { + assert (aggregationTrees != null); + for (ASTNode value : aggregationTrees.values()) { + // 4.1 Determine type of UDAF + // This is the GenericUDAF name + String aggName = unescapeIdentifier(value.getChild(0).getText()); + boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI; + boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR; + if (isDistinct) { +// continue; + numDistinctUDFs++; + } - // 4.2 Determine type of UDF - // This is the GenericUDAF name - String aggName = unescapeIdentifier(value.getChild(0).getText()); - boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI; - boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR; + // 4.2 Convert UDAF Params to ExprNodeDesc + ArrayList aggParameters = new ArrayList(); + for (int i = 1; i < value.getChildCount(); i++) { + ASTNode paraExpr = (ASTNode) value.getChild(i); + ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, groupByInputRowResolver); + aggParameters.add(paraExprNode); + } - if (isDistinct) { - numDistinctUDFs++; + Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, isDistinct); + GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator(aggName, + aggParameters, value, isDistinct, isAllColumns); + assert (genericUDAFEvaluator != null); + GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); + AggInfo aInfo = new AggInfo(aggParameters, udaf.returnType, aggName, isDistinct); + aggregations.add(aInfo); + String field = getColumnInternalName(gbExprNDescLst.size() + aggregations.size() - 1); + outputColumnNames.add(field); + groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.m_returnType, + "", false)); } - Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, - isDistinct); - GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator( - aggName, aggParameters, value, isDistinct, isAllColumns); - assert (genericUDAFEvaluator != null); - GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, - amode, aggParameters); - AggInfo aInfo = new AggInfo(aggParameters, udaf.returnType, aggName, - isDistinct); - aggregations.add(aInfo); - String field = getColumnInternalName(gbExprNDescLst.size() - + aggregations.size() - 1); - outputColumnNames.add(field); - groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, - aInfo.m_returnType, "", false)); } gbRel = genGBRelNode(gbExprNDescLst, aggregations, srcRel);