diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java index bb20c9b047..07350e26b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java @@ -42,8 +42,8 @@ import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; import org.apache.hadoop.hive.ql.parse.UnparseTranslator; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -293,7 +293,7 @@ private static String getDefaultValue(ASTNode node, ASTNode typeChild, TokenRewr throws SemanticException{ // first create expression from defaultValueAST TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null); - ExprNodeDesc defaultValExpr = TypeCheckProcFactory.genExprNode(node, typeCheckCtx).get(node); + ExprNodeDesc defaultValExpr = ExprNodeTypeCheck.genExprNode(node, typeCheckCtx).get(node); if (defaultValExpr == null) { throw new SemanticException(ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("Invalid Default value!")); @@ -445,7 +445,7 @@ public static void validateCheckConstraint(List columns, List genExprs = TypeCheckProcFactory.genExprNode(checkExprAST, typeCheckCtx); + Map genExprs = ExprNodeTypeCheck.genExprNode(checkExprAST, typeCheckCtx); ExprNodeDesc checkExpr = genExprs.get(checkExprAST); if (checkExpr == null) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/drop/AbstractDropPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/drop/AbstractDropPartitionAnalyzer.java index ae5048d527..3b8cb25943 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/drop/AbstractDropPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/drop/AbstractDropPartitionAnalyzer.java @@ -49,8 +49,8 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; @@ -158,7 +158,7 @@ protected void analyzeCommand(TableName tableName, Map partition ASTNode partValNode = (ASTNode)partSpecSingleKey.getChild(2); TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null); ExprNodeConstantDesc valExpr = - (ExprNodeConstantDesc)TypeCheckProcFactory.genExprNode(partValNode, typeCheckCtx).get(partValNode); + (ExprNodeConstantDesc) ExprNodeTypeCheck.genExprNode(partValNode, typeCheckCtx).get(partValNode); Object val = valExpr.getValue(); boolean isDefaultPartitionName = val.equals(defaultPartitionName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java index ab86a80217..b5742c6ecd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java @@ -29,9 +29,8 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.serde2.SerDeException; /** * Operator factory for pruning processing of operator graph We find @@ -117,8 +116,8 @@ protected void addPruningPred(Map opToPrunner, ExprNodeDesc pruner_pred = null; if (old_pruner_pred != null) { // or the old_pruner_pred and the new_ppr_pred - pruner_pred = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("OR", - old_pruner_pred, new_pruner_pred); + pruner_pred = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor().getFuncExprNodeDesc( + "OR", old_pruner_pred, new_pruner_pred); } else { pruner_pred = new_pruner_pred; } @@ -153,8 +152,8 @@ protected void addPruningPred(Map> ExprNodeDesc old_pruner_pred = oldPartToPruner.get(part.getName()); if (old_pruner_pred != null) { // or the old_pruner_pred and the new_ppr_pred - pruner_pred = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("OR", - old_pruner_pred, new_pruner_pred); + pruner_pred = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor().getFuncExprNodeDesc( + "OR", old_pruner_pred, new_pruner_pred); } else { pruner_pred = new_pruner_pred; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 9ec0d73916..c6798e9933 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -59,8 +58,8 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ColStatistics; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -234,7 +233,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (numBuckets > 0) { bucketColumns = new ArrayList<>(); //add a cast(ROW__ID as int) to wrap in UDFToInteger() - bucketColumns.add(ParseUtils.createConversionCast(new ExprNodeColumnDesc(ci), TypeInfoFactory.intTypeInfo)); + bucketColumns.add(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(new ExprNodeColumnDesc(ci), TypeInfoFactory.intTypeInfo)); } } else { if (!destTable.getSortCols().isEmpty()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 63882fd57a..ba6eefb2d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -70,9 +70,9 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter.HiveNlsString.Interpretation; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -80,6 +80,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeSubQueryDesc; +import org.apache.hadoop.hive.ql.plan.SubqueryType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; @@ -219,8 +220,8 @@ private void throwInvalidSubqueryError(final ASTNode comparisonOp) throws Semant // <>ANY and =ALL is not supported private RexNode convertSubquerySomeAll(final ExprNodeSubQueryDesc subQueryDesc) throws SemanticException { - assert(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.SOME - || subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.ALL); + assert(subQueryDesc.getType() == SubqueryType.SOME + || subQueryDesc.getType() == SubqueryType.ALL); RexNode rexNodeLhs = convert(subQueryDesc.getSubQueryLhs()); ASTNode comparisonOp = subQueryDesc.getComparisonOp(); @@ -228,7 +229,7 @@ private RexNode convertSubquerySomeAll(final ExprNodeSubQueryDesc subQueryDesc) switch (comparisonOp.getType()) { case HiveParser.EQUAL: - if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.ALL) { + if(subQueryDesc.getType() == SubqueryType.ALL) { throwInvalidSubqueryError(comparisonOp); } quantifyOperator = SqlStdOperatorTable.SOME_EQ; @@ -246,7 +247,7 @@ private RexNode convertSubquerySomeAll(final ExprNodeSubQueryDesc subQueryDesc) quantifyOperator = SqlStdOperatorTable.SOME_GE; break; case HiveParser.NOTEQUAL: - if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.SOME) { + if(subQueryDesc.getType() == SubqueryType.SOME) { throwInvalidSubqueryError(comparisonOp); } quantifyOperator = SqlStdOperatorTable.SOME_NE; @@ -256,19 +257,19 @@ private RexNode convertSubquerySomeAll(final ExprNodeSubQueryDesc subQueryDesc) "Invalid operator:" + comparisonOp.toString())); } - if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.ALL) { + if(subQueryDesc.getType() == SubqueryType.ALL) { quantifyOperator = SqlStdOperatorTable.some(quantifyOperator.comparisonKind.negateNullSafe()); } RexNode someQuery = getSomeSubquery(subQueryDesc.getRexSubQuery(), rexNodeLhs, quantifyOperator); - if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.ALL) { + if(subQueryDesc.getType() == SubqueryType.ALL) { return cluster.getRexBuilder().makeCall(SqlStdOperatorTable.NOT, someQuery); } return someQuery; } private RexNode convert(final ExprNodeSubQueryDesc subQueryDesc) throws SemanticException { - if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.IN) { + if(subQueryDesc.getType() == SubqueryType.IN) { /* * Check.5.h :: For In and Not In the SubQuery must implicitly or * explicitly only contain one select item. @@ -284,10 +285,10 @@ private RexNode convert(final ExprNodeSubQueryDesc subQueryDesc) throws Semanti RexNode rexSubQuery = RexSubQuery.in(subQueryDesc.getRexSubQuery(), ImmutableList.of(rexNodeLhs)); return rexSubQuery; - } else if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.EXISTS) { + } else if(subQueryDesc.getType() == SubqueryType.EXISTS) { RexNode subQueryNode = RexSubQuery.exists(subQueryDesc.getRexSubQuery()); return subQueryNode; - } else if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.SCALAR){ + } else if(subQueryDesc.getType() == SubqueryType.SCALAR){ if(subQueryDesc.getRexSubQuery().getRowType().getFieldCount() > 1) { throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg( "SubQuery can contain only 1 item in Select List.")); @@ -295,8 +296,8 @@ private RexNode convert(final ExprNodeSubQueryDesc subQueryDesc) throws Semanti //create RexSubQuery node RexNode rexSubQuery = RexSubQuery.scalar(subQueryDesc.getRexSubQuery()); return rexSubQuery; - } else if(subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.SOME - || subQueryDesc.getType() == ExprNodeSubQueryDesc.SubqueryType.ALL) { + } else if(subQueryDesc.getType() == SubqueryType.SOME + || subQueryDesc.getType() == SubqueryType.ALL) { return convertSubquerySomeAll(subQueryDesc); } else { throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg( @@ -382,13 +383,15 @@ private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException { // For compare, we will convert requisite children // For BETWEEN skip the first child (the revert boolean) if (!isBetween || i > 0) { - tmpExprNode = ParseUtils.createConversionCast(childExpr, (PrimitiveTypeInfo) tgtDT); + tmpExprNode = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(childExpr, (PrimitiveTypeInfo) tgtDT); } } else if (isNumeric) { // For numeric, we'll do minimum necessary cast - if we cast to the type // of expression, bad things will happen. PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(childExpr, tgtDT); - tmpExprNode = ParseUtils.createConversionCast(childExpr, minArgType); + tmpExprNode = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(childExpr, minArgType); } else { throw new AssertionError("Unexpected " + tgtDT + " - not a numeric op or compare"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveUnionVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveUnionVisitor.java index 9ad5eebb27..ce58e6b043 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveUnionVisitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveUnionVisitor.java @@ -31,8 +31,8 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.opconventer.HiveOpConverter.OpAttr; -import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -128,7 +128,8 @@ OpAttr visit(HiveUnion unionRel) throws SemanticException { ExprNodeDesc column = new ExprNodeColumnDesc(oInfo.getType(), oInfo.getInternalName(), oInfo.getTabAlias(), oInfo.getIsVirtualCol(), oInfo.isSkewedCol()); if (!oInfo.getType().equals(uInfo.getType())) { - column = ParseUtils.createConversionCast(column, (PrimitiveTypeInfo) uInfo.getType()); + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, (PrimitiveTypeInfo) uInfo.getType()); } columns.add(column); colName.add(uInfo.getInternalName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index b0ceecb24e..ee64dbf836 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -81,6 +81,9 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FetchWork; @@ -1497,7 +1500,7 @@ private static boolean getPartExprNodeDesc(ASTNode astNode, HiveConf conf, ASTNode partVal = (ASTNode)childASTNode.getChildren().get(1); if (!defaultPartitionName.equalsIgnoreCase(unescapeSQLString(partVal.getText()))) { astExprNodeMap.put((ASTNode)childASTNode.getChildren().get(0), - TypeCheckProcFactory.genExprNode(partVal, typeCheckCtx).get(partVal)); + ExprNodeTypeCheck.genExprNode(partVal, typeCheckCtx).get(partVal)); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index cde8eada64..773c5ca811 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -226,8 +226,9 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.MaterializedViewRewritingRelVisitor; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter; -import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinCondTypeCheckProcFactory; -import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinTypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.JoinCondTypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.JoinTypeCheckCtx; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.PlanModifierForReturnPath; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter; @@ -243,6 +244,8 @@ import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -2689,8 +2692,8 @@ private RelNode genJoinRelNode(RelNode leftRel, String leftTableAlias, RelNode r } else if (unparseTranslator != null && unparseTranslator.isEnabled()) { genAllExprNodeDesc(joinCond, input, jCtx); } - Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, - jCtx); + Map exprNodes = ExprNodeTypeCheck.genExprNodeJoinCond( + joinCond, jCtx); if (jCtx.getError() != null) { throw new SemanticException(SemanticAnalyzer.generateErrorMessage(jCtx.getErrorSrcNode(), jCtx.getError())); @@ -4542,7 +4545,7 @@ private void setQueryHints(QB qb) throws SemanticException { ASTNode expr = (ASTNode) selExprList.getChild(posn).getChild(0); int exprType = expr.getType(); if (exprType == HiveParser.TOK_FUNCTION || exprType == HiveParser.TOK_FUNCTIONSTAR) { - String funcName = TypeCheckProcFactory.DefaultExprProcessor.getFunctionText(expr, true); + String funcName = TypeCheckProcFactory.getFunctionText(expr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); if (fi != null && fi.getGenericUDTF() != null) { LOG.debug("Find UDTF " + funcName); @@ -5256,4 +5259,5 @@ public static void initializeMetadataProviderClass() { NATIVE, JDBC } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 81f35cec84..9bcc472a62 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -276,7 +277,7 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2a2359402b..c4876feff7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -200,6 +200,9 @@ import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -998,7 +1001,7 @@ private void transformWithinGroup(ASTNode expressionTree, Tree withinGroupNode) return exprs; } - static String generateErrorMessage(ASTNode ast, String message) { + public static String generateErrorMessage(ASTNode ast, String message) { StringBuilder sb = new StringBuilder(); if (ast == null) { sb.append(message).append(". Cannot tell the position of null AST."); @@ -4440,8 +4443,7 @@ boolean isRegex(String pattern, HiveConf conf) { int udtfExprType = udtfExpr.getType(); if (udtfExprType == HiveParser.TOK_FUNCTION || udtfExprType == HiveParser.TOK_FUNCTIONSTAR) { - String funcName = TypeCheckProcFactory.DefaultExprProcessor - .getFunctionText(udtfExpr, true); + String funcName = TypeCheckProcFactory.getFunctionText(udtfExpr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); if (fi != null) { genericUDTF = fi.getGenericUDTF(); @@ -4688,7 +4690,7 @@ private RowResolver getColForInsertStmtSpec(Map targetCol2 try { ASTNode defValAst = parseDriver.parseExpression(defaultValue); - exp = TypeCheckProcFactory.genExprNode(defValAst, new TypeCheckCtx(null)).get(defValAst); + exp = ExprNodeTypeCheck.genExprNode(defValAst, new TypeCheckCtx(null)).get(defValAst); } catch(Exception e) { throw new SemanticException("Error while parsing default value: " + defaultValue + ". Error message: " + e.getMessage()); @@ -7121,17 +7123,17 @@ private ExprNodeDesc getCheckConstraintExpr(Table tbl, Operator input, RowResolv ASTNode checkExprAST = parseDriver.parseExpression(checkExprStr); //replace column references in checkExprAST with corresponding columns in input replaceColumnReference(checkExprAST, col2Cols, inputRR); - Map genExprs = TypeCheckProcFactory + Map genExprs = ExprNodeTypeCheck .genExprNode(checkExprAST, typeCheckCtx); ExprNodeDesc checkExpr = genExprs.get(checkExprAST); // Check constraint fails only if it evaluates to false, NULL/UNKNOWN should evaluate to TRUE - ExprNodeDesc notFalseCheckExpr = TypeCheckProcFactory.DefaultExprProcessor. + ExprNodeDesc notFalseCheckExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(). getFuncExprNodeDesc("isnotfalse", checkExpr); if(checkAndExprs == null) { checkAndExprs = notFalseCheckExpr; } else { - checkAndExprs = TypeCheckProcFactory.DefaultExprProcessor. + checkAndExprs = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(). getFuncExprNodeDesc("and", checkAndExprs, notFalseCheckExpr); } } catch(Exception e) { @@ -7212,8 +7214,8 @@ else if(dest_type == QBMetaData.DEST_PARTITION){ ExprNodeDesc combinedConstraintExpr = null; if(nullConstraintExpr != null && checkConstraintExpr != null) { assert (input.getParentOperators().size() == 1); - combinedConstraintExpr = TypeCheckProcFactory.DefaultExprProcessor. - getFuncExprNodeDesc("and", nullConstraintExpr, checkConstraintExpr); + combinedConstraintExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("and", nullConstraintExpr, checkConstraintExpr); } else if(nullConstraintExpr != null) { @@ -7224,8 +7226,8 @@ else if(checkConstraintExpr != null) { } if (combinedConstraintExpr != null) { - ExprNodeDesc constraintUDF = TypeCheckProcFactory.DefaultExprProcessor. - getFuncExprNodeDesc("enforce_constraint", combinedConstraintExpr); + ExprNodeDesc constraintUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("enforce_constraint", combinedConstraintExpr); return putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(constraintUDF, false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); @@ -7264,12 +7266,12 @@ private ExprNodeDesc getNotNullConstraintExpr(Table targetTable, Operator input, continue; } if (nullConstraintBitSet.indexOf(constraintIdx) != -1) { - ExprNodeDesc currExpr = TypeCheckProcFactory.toExprNodeDesc(colInfos.get(colExprIdx)); - ExprNodeDesc isNotNullUDF = TypeCheckProcFactory.DefaultExprProcessor. - getFuncExprNodeDesc("isnotnull", currExpr); + ExprNodeDesc currExpr = ExprNodeTypeCheck.toExprNodeDesc(colInfos.get(colExprIdx)); + ExprNodeDesc isNotNullUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("isnotnull", currExpr); if (currUDF != null) { - currUDF = TypeCheckProcFactory.DefaultExprProcessor. - getFuncExprNodeDesc("and", currUDF, isNotNullUDF); + currUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("and", currUDF, isNotNullUDF); } else { currUDF = isNotNullUDF; } @@ -8443,8 +8445,8 @@ private Operator genConversionSelectOperator(String dest, QB qb, Operator input, // cannot convert to complex types column = null; } else { - column = ParseUtils.createConversionCast( - column, (PrimitiveTypeInfo)tableFieldTypeInfo); + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, (PrimitiveTypeInfo)tableFieldTypeInfo); } if (column == null) { String reason = "Cannot convert column " + i + " from " @@ -8661,7 +8663,8 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowField.getInternalName(), rowField.getTabAlias(), true); if (convert) { - column = ParseUtils.createConversionCast(column, TypeInfoFactory.intTypeInfo); + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, TypeInfoFactory.intTypeInfo); } List rlist = new ArrayList(1); rlist.add(column); @@ -8701,8 +8704,8 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, // cannot convert to complex types column = null; } else { - column = ParseUtils.createConversionCast( - column, (PrimitiveTypeInfo)tableFieldTypeInfo); + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, (PrimitiveTypeInfo)tableFieldTypeInfo); } if (column == null) { String reason = "Cannot convert column " + posn + " from " @@ -9540,8 +9543,8 @@ private Operator genMapGroupByForSemijoin(List fields, Operator inpu for (int i = 0; i < keys.length; i++) { if (TypeInfoUtils.isConversionRequiredForComparison( keys[i][k].getTypeInfo(), commonType)) { - keys[i][k] = ParseUtils.createConversionCast( - keys[i][k], (PrimitiveTypeInfo)commonType); + keys[i][k] = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(keys[i][k], (PrimitiveTypeInfo)commonType); } else { // For the case no implicit type conversion, e.g., varchar(5) and varchar(10), // pick the common type for all the keys since during run-time, same key type is assumed. @@ -11130,7 +11133,7 @@ private Operator genUnionPlan(String unionalias, String leftalias, Iterator oIter = origInputFieldMap.values().iterator(); Iterator uIter = fieldMap.values().iterator(); - List columns = new ArrayList(); + List columns = new ArrayList<>(); boolean needsCast = false; while (oIter.hasNext()) { ColumnInfo oInfo = oIter.next(); @@ -11139,8 +11142,8 @@ private Operator genUnionPlan(String unionalias, String leftalias, oInfo.getTabAlias(), oInfo.getIsVirtualCol(), oInfo.isSkewedCol()); if (!oInfo.getType().equals(uInfo.getType())) { needsCast = true; - column = ParseUtils.createConversionCast( - column, (PrimitiveTypeInfo)uInfo.getType()); + column = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(column, (PrimitiveTypeInfo)uInfo.getType()); } columns.add(column); } @@ -11233,14 +11236,14 @@ private ExprNodeDesc genSamplePredicate(TableSample ts, TypeInfoFactory.intTypeInfo, bucketingVersion == 2 ? new GenericUDFMurmurHash() : new GenericUDFHash(), args); LOG.info("hashfnExpr = " + hashfnExpr); - ExprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc andExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr); LOG.info("andExpr = " + andExpr); - ExprNodeDesc modExpr = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc modExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("%", andExpr, denominatorExpr); LOG.info("modExpr = " + modExpr); LOG.info("numeratorExpr = " + numeratorExpr); - equalsExpr = TypeCheckProcFactory.DefaultExprProcessor + equalsExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("==", modExpr, numeratorExpr); LOG.info("equalsExpr = " + equalsExpr); } @@ -11433,13 +11436,13 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { new RowSchema(rwsch.getColumnInfos()), top); } } else { - boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE); + boolean testMode = conf.getBoolVar(ConfVars.HIVETESTMODE); if (testMode) { String tabName = tab.getTableName(); // has the user explicitly asked not to sample this table String unSampleTblList = conf - .getVar(HiveConf.ConfVars.HIVETESTMODENOSAMPLE); + .getVar(ConfVars.HIVETESTMODENOSAMPLE); String[] unSampleTbls = unSampleTblList.split(","); boolean unsample = false; for (String unSampleTbl : unSampleTbls) { @@ -11468,14 +11471,14 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { LOG.info("No need for sample filter"); } else { // The table is not bucketed, add a dummy filter :: rand() - int freq = conf.getIntVar(HiveConf.ConfVars.HIVETESTMODESAMPLEFREQ); + int freq = conf.getIntVar(ConfVars.HIVETESTMODESAMPLEFREQ); TableSample tsSample = new TableSample(1, freq); tsSample.setInputPruning(false); qb.getParseInfo().setTabSample(alias, tsSample); LOG.info("Need sample filter"); - ExprNodeDesc randFunc = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("rand", new ExprNodeConstantDesc(Integer - .valueOf(460476415))); + ExprNodeDesc randFunc = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("rand", + new ExprNodeConstantDesc(Integer.valueOf(460476415))); ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false, alias, rwsch, randFunc, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePred, true); @@ -13011,7 +13014,7 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) tcCtx.setUnparseTranslator(unparseTranslator); Map nodeOutputs = - TypeCheckProcFactory.genExprNode(expr, tcCtx); + ExprNodeTypeCheck.genExprNode(expr, tcCtx); ExprNodeDesc desc = nodeOutputs.get(expr); if (desc == null) { String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr @@ -13076,7 +13079,7 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) return nodeOutputs; } - private Map translateFieldDesc(ASTNode node) { + protected final Map translateFieldDesc(ASTNode node) { Map map = new HashMap<>(); if (node.getType() == HiveParser.DOT) { for (Node child : node.getChildren()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java deleted file mode 100644 index e0f4826ed7..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ /dev/null @@ -1,1843 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; - -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; -import org.apache.hadoop.hive.common.type.Date; -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.common.type.TimestampTZUtil; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FunctionInfo; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; -import org.apache.hadoop.hive.ql.lib.CostLessRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.SubqueryExpressionWalker; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; -import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException; -import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; -import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeSubQueryDesc; -import org.apache.hadoop.hive.ql.udf.SettableUDF; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.apache.hadoop.io.NullWritable; -import org.apache.hive.common.util.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.SetMultimap; - -/** - * The Factory for creating typecheck processors. The typecheck processors are - * used to processes the syntax trees for expressions and convert them into - * expression Node Descriptor trees. They also introduce the correct conversion - * functions to do proper implicit conversion. - */ -public class TypeCheckProcFactory { - - protected static final Logger LOG = LoggerFactory.getLogger(TypeCheckProcFactory.class - .getName()); - - protected TypeCheckProcFactory() { - // prevent instantiation - } - - /** - * Function to do groupby subexpression elimination. This is called by all the - * processors initially. As an example, consider the query select a+b, - * count(1) from T group by a+b; Then a+b is already precomputed in the group - * by operators key, so we substitute a+b in the select list with the internal - * column name of the a+b expression that appears in the in input row - * resolver. - * - * @param nd - * The node that is being inspected. - * @param procCtx - * The processor context. - * - * @return exprNodeColumnDesc. - */ - public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) - throws SemanticException { - // We recursively create the exprNodeDesc. Base cases: when we encounter - // a column ref, we convert that into an exprNodeColumnDesc; when we - // encounter - // a constant, we convert that into an exprNodeConstantDesc. For others we - // just - // build the exprNodeFuncDesc with recursively built children. - ASTNode expr = (ASTNode) nd; - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - - // bypass only if outerRR is not null. Otherwise we need to look for expressions in outerRR for - // subqueries e.g. select min(b.value) from table b group by b.key - // having key in (select .. where a = min(b.value) - if (!ctx.isUseCaching() && ctx.getOuterRR() == null) { - return null; - } - - RowResolver input = ctx.getInputRR(); - ExprNodeDesc desc = null; - - if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) { - return null; - } - - // If the current subExpression is pre-calculated, as in Group-By etc. - ColumnInfo colInfo = input.getExpression(expr); - - // try outer row resolver - RowResolver outerRR = ctx.getOuterRR(); - if(colInfo == null && outerRR != null) { - colInfo = outerRR.getExpression(expr); - } - if (colInfo != null) { - desc = new ExprNodeColumnDesc(colInfo); - ASTNode source = input.getExpressionSource(expr); - if (source != null && ctx.getUnparseTranslator() != null) { - ctx.getUnparseTranslator().addCopyTranslation(expr, source); - } - return desc; - } - return desc; - } - - public static Map genExprNode(ASTNode expr, TypeCheckCtx tcCtx) - throws SemanticException { - return genExprNode(expr, tcCtx, new TypeCheckProcFactory()); - } - - protected static Map genExprNode(ASTNode expr, - TypeCheckCtx tcCtx, TypeCheckProcFactory tf) throws SemanticException { - // Create the walker, the rules dispatcher and the context. - // create a walker which walks the tree in a DFS manner while maintaining - // the operator stack. The dispatcher - // generates the plan from the operator tree - - SetMultimap astNodeToProcessor = HashMultimap.create(); - astNodeToProcessor.put(HiveParser.TOK_NULL, tf.getNullExprProcessor()); - - astNodeToProcessor.put(HiveParser.Number, tf.getNumExprProcessor()); - astNodeToProcessor.put(HiveParser.IntegralLiteral, tf.getNumExprProcessor()); - astNodeToProcessor.put(HiveParser.NumberLiteral, tf.getNumExprProcessor()); - - astNodeToProcessor.put(HiveParser.Identifier, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.StringLiteral, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_CHARSETLITERAL, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_STRINGLITERALSEQUENCE, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_IF, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_CASE, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_WHEN, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_IN, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_ARRAY, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_MAP, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_STRUCT, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_EXISTS, tf.getStrExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_SUBQUERY_OP_NOTIN, tf.getStrExprProcessor()); - - astNodeToProcessor.put(HiveParser.KW_TRUE, tf.getBoolExprProcessor()); - astNodeToProcessor.put(HiveParser.KW_FALSE, tf.getBoolExprProcessor()); - - astNodeToProcessor.put(HiveParser.TOK_DATELITERAL, tf.getDateTimeExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_TIMESTAMPLITERAL, tf.getDateTimeExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_TIMESTAMPLOCALTZLITERAL, tf.getDateTimeExprProcessor()); - - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_YEAR_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_MONTH_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_DAY_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_HOUR_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_MINUTE_LITERAL, tf.getIntervalExprProcessor()); - astNodeToProcessor.put(HiveParser.TOK_INTERVAL_SECOND_LITERAL, tf.getIntervalExprProcessor()); - - astNodeToProcessor.put(HiveParser.TOK_TABLE_OR_COL, tf.getColumnExprProcessor()); - - astNodeToProcessor.put(HiveParser.TOK_SUBQUERY_EXPR, tf.getSubQueryExprProcessor()); - - // The dispatcher fires the processor corresponding to the closest matching - // rule and passes the context along - Dispatcher disp = new CostLessRuleDispatcher(tf.getDefaultExprProcessor(), - astNodeToProcessor, tcCtx); - GraphWalker ogw = new SubqueryExpressionWalker(disp); - - // Create a list of top nodes - ArrayList topNodes = Lists.newArrayList(expr); - HashMap nodeOutputs = new LinkedHashMap(); - ogw.startWalking(topNodes, nodeOutputs); - - return convert(nodeOutputs); - } - - // temporary type-safe casting - private static Map convert(Map outputs) { - Map converted = new LinkedHashMap(); - for (Map.Entry entry : outputs.entrySet()) { - if (entry.getKey() instanceof ASTNode && - (entry.getValue() == null || entry.getValue() instanceof ExprNodeDesc)) { - converted.put((ASTNode)entry.getKey(), (ExprNodeDesc)entry.getValue()); - } else { - LOG.warn("Invalid type entry " + entry); - } - } - return converted; - } - - /** - * Processor for processing NULL expression. - */ - public static class NullExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - return new ExprNodeConstantDesc(TypeInfoFactory. - getPrimitiveTypeInfoFromPrimitiveWritable(NullWritable.class), null); - } - - } - - /** - * Factory method to get NullExprProcessor. - * - * @return NullExprProcessor. - */ - public NullExprProcessor getNullExprProcessor() { - return new NullExprProcessor(); - } - - /** - * Processor for processing numeric constants. - */ - public static class NumExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - Number v = null; - ExprNodeConstantDesc decimalNode = null; - ASTNode expr = (ASTNode) nd; - // The expression can be any one of Double, Long and Integer. We - // try to parse the expression in that order to ensure that the - // most specific type is used for conversion. - try { - if (expr.getText().endsWith("L")) { - // Literal bigint. - v = Long.valueOf(expr.getText().substring(0, expr.getText().length() - 1)); - } else if (expr.getText().endsWith("S")) { - // Literal smallint. - v = Short.valueOf(expr.getText().substring(0, expr.getText().length() - 1)); - } else if (expr.getText().endsWith("Y")) { - // Literal tinyint. - v = Byte.valueOf(expr.getText().substring(0, expr.getText().length() - 1)); - } else if (expr.getText().endsWith("BD")) { - // Literal decimal - String strVal = expr.getText().substring(0, expr.getText().length() - 2); - return createDecimal(strVal, false); - } else if (expr.getText().endsWith("F")) { - // Literal float. - v = Float.valueOf(expr.getText().substring(0, expr.getText().length() - 1)); - } else if (expr.getText().endsWith("D")) { - // Literal double. - v = Double.valueOf(expr.getText().substring(0, expr.getText().length() - 1)); - } else { - v = Double.valueOf(expr.getText()); - if (expr.getText() != null && !expr.getText().toLowerCase().contains("e")) { - decimalNode = createDecimal(expr.getText(), true); - if (decimalNode != null) { - v = null; // We will use decimal if all else fails. - } - } - v = Long.valueOf(expr.getText()); - v = Integer.valueOf(expr.getText()); - } - } catch (NumberFormatException e) { - // do nothing here, we will throw an exception in the following block - } - if (v == null && decimalNode == null) { - throw new SemanticException(ErrorMsg.INVALID_NUMERICAL_CONSTANT.getMsg(expr)); - } - return v != null ? new ExprNodeConstantDesc(v) : decimalNode; - } - - public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) { - HiveDecimal hd = HiveDecimal.create(strVal); - if (notNull && hd == null) { - return null; - } - return new ExprNodeConstantDesc(adjustType(hd), hd); - } - - private static DecimalTypeInfo adjustType(HiveDecimal hd) { - // Note: the normalize() call with rounding in HiveDecimal will currently reduce the - // precision and scale of the value by throwing away trailing zeroes. This may or may - // not be desirable for the literals; however, this used to be the default behavior - // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. - int prec = 1; - int scale = 0; - if (hd != null) { - prec = hd.precision(); - scale = hd.scale(); - } - DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale); - return typeInfo; - } - - } - - /** - * Factory method to get NumExprProcessor. - * - * @return NumExprProcessor. - */ - public NumExprProcessor getNumExprProcessor() { - return new NumExprProcessor(); - } - - /** - * Processor for processing string constants. - */ - public static class StrExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - ASTNode expr = (ASTNode) nd; - String str = null; - - switch (expr.getToken().getType()) { - case HiveParser.StringLiteral: - str = BaseSemanticAnalyzer.unescapeSQLString(expr.getText()); - break; - case HiveParser.TOK_STRINGLITERALSEQUENCE: - StringBuilder sb = new StringBuilder(); - for (Node n : expr.getChildren()) { - sb.append( - BaseSemanticAnalyzer.unescapeSQLString(((ASTNode)n).getText())); - } - str = sb.toString(); - break; - case HiveParser.TOK_CHARSETLITERAL: - str = BaseSemanticAnalyzer.charSetString(expr.getChild(0).getText(), - expr.getChild(1).getText()); - break; - default: - // HiveParser.identifier | HiveParse.KW_IF | HiveParse.KW_LEFT | - // HiveParse.KW_RIGHT - str = BaseSemanticAnalyzer.unescapeIdentifier(expr.getText().toLowerCase()); - break; - } - return new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, str); - } - - } - - /** - * Factory method to get StrExprProcessor. - * - * @return StrExprProcessor. - */ - public StrExprProcessor getStrExprProcessor() { - return new StrExprProcessor(); - } - - /** - * Processor for boolean constants. - */ - public static class BoolExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - ASTNode expr = (ASTNode) nd; - Boolean bool = null; - - switch (expr.getToken().getType()) { - case HiveParser.KW_TRUE: - bool = Boolean.TRUE; - break; - case HiveParser.KW_FALSE: - bool = Boolean.FALSE; - break; - default: - assert false; - } - return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, bool); - } - - } - - /** - * Factory method to get BoolExprProcessor. - * - * @return BoolExprProcessor. - */ - public BoolExprProcessor getBoolExprProcessor() { - return new BoolExprProcessor(); - } - - /** - * Processor for date constants. - */ - public static class DateTimeExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - ASTNode expr = (ASTNode) nd; - String timeString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); - - // Get the string value and convert to a Date value. - try { - // todo replace below with joda-time, which supports timezone - if (expr.getType() == HiveParser.TOK_DATELITERAL) { - PrimitiveTypeInfo typeInfo = TypeInfoFactory.dateTypeInfo; - return new ExprNodeConstantDesc(typeInfo, - Date.valueOf(timeString)); - } - if (expr.getType() == HiveParser.TOK_TIMESTAMPLITERAL) { - return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, - Timestamp.valueOf(timeString)); - } - if (expr.getType() == HiveParser.TOK_TIMESTAMPLOCALTZLITERAL) { - HiveConf conf; - try { - conf = Hive.get().getConf(); - } catch (HiveException e) { - throw new SemanticException(e); - } - return new ExprNodeConstantDesc(TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), - TimestampTZUtil.parse(timeString)); - } - throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); - } catch (Exception err) { - throw new SemanticException( - "Unable to convert time literal '" + timeString + "' to time value.", err); - } - } - } - - /** - * Processor for interval constants. - */ - public static class IntervalExprProcessor implements NodeProcessor { - - private static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(DateUtils.NANOS_PER_SEC); - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - ASTNode expr = (ASTNode) nd; - String intervalString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); - - // Get the string value and convert to a Interval value. - try { - switch (expr.getType()) { - case HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, - HiveIntervalYearMonth.valueOf(intervalString)); - case HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, - HiveIntervalDayTime.valueOf(intervalString)); - case HiveParser.TOK_INTERVAL_YEAR_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, - new HiveIntervalYearMonth(Integer.parseInt(intervalString), 0)); - case HiveParser.TOK_INTERVAL_MONTH_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, - new HiveIntervalYearMonth(0, Integer.parseInt(intervalString))); - case HiveParser.TOK_INTERVAL_DAY_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, - new HiveIntervalDayTime(Integer.parseInt(intervalString), 0, 0, 0, 0)); - case HiveParser.TOK_INTERVAL_HOUR_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, - new HiveIntervalDayTime(0, Integer.parseInt(intervalString), 0, 0, 0)); - case HiveParser.TOK_INTERVAL_MINUTE_LITERAL: - return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, - new HiveIntervalDayTime(0, 0, Integer.parseInt(intervalString), 0, 0)); - case HiveParser.TOK_INTERVAL_SECOND_LITERAL: - BigDecimal bd = new BigDecimal(intervalString); - BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger()); - BigDecimal bdNanos = bd.subtract(bdSeconds); - return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, - new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(), - bdNanos.multiply(NANOS_PER_SEC_BD).intValue())); - default: - throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); - } - } catch (Exception err) { - throw new SemanticException( - "Unable to convert interval literal '" + intervalString + "' to interval value.", err); - } - } - } - - /** - * Factory method to get IntervalExprProcessor. - * - * @return IntervalExprProcessor. - */ - public IntervalExprProcessor getIntervalExprProcessor() { - return new IntervalExprProcessor(); - } - - /** - * Factory method to get DateExprProcessor. - * - * @return DateExprProcessor. - */ - public DateTimeExprProcessor getDateTimeExprProcessor() { - return new DateTimeExprProcessor(); - } - - /** - * Processor for table columns. - */ - public static class ColumnExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - ASTNode expr = (ASTNode) nd; - ASTNode parent = stack.size() > 1 ? (ASTNode) stack.get(stack.size() - 2) : null; - RowResolver input = ctx.getInputRR(); - if(input == null) { - ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr); - return null; - } - - if (expr.getType() != HiveParser.TOK_TABLE_OR_COL) { - ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr); - return null; - } - - assert (expr.getChildCount() == 1); - String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr - .getChild(0).getText()); - - boolean isTableAlias = input.hasTableAlias(tableOrCol); - ColumnInfo colInfo = null; - try { - colInfo = input.get(null, tableOrCol); - } catch (SemanticException semanticException) { - if (!isTableAlias || parent == null || parent.getType() != HiveParser.DOT) { - throw semanticException; - } - } - // try outer row resolver - if(ctx.getOuterRR() != null && colInfo == null && !isTableAlias) { - RowResolver outerRR = ctx.getOuterRR(); - isTableAlias = outerRR.hasTableAlias(tableOrCol); - colInfo = outerRR.get(null, tableOrCol); - } - - if (isTableAlias) { - if (colInfo != null) { - if (parent != null && parent.getType() == HiveParser.DOT) { - // It's a table alias. - return null; - } - // It's a column. - return toExprNodeDesc(colInfo); - } else { - // It's a table alias. - // We will process that later in DOT. - return null; - } - } else { - if (colInfo == null) { - // It's not a column or a table alias. - if (input.getIsExprResolver()) { - ASTNode exprNode = expr; - if (!stack.empty()) { - ASTNode tmp = (ASTNode) stack.pop(); - if (!stack.empty()) { - exprNode = (ASTNode) stack.peek(); - } - stack.push(tmp); - } - ctx.setError(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(exprNode), expr); - return null; - } else { - List possibleColumnNames = input.getReferenceableColumnAliases(tableOrCol, -1); - String reason = String.format("(possible column names are: %s)", - StringUtils.join(possibleColumnNames, ", ")); - ctx.setError(ErrorMsg.INVALID_TABLE_OR_COLUMN.getMsg(expr.getChild(0), reason), - expr); - LOG.debug(ErrorMsg.INVALID_TABLE_OR_COLUMN.toString() + ":" - + input.toString()); - return null; - } - } else { - // It's a column. - return toExprNodeDesc(colInfo); - } - } - - } - - } - - static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) { - ObjectInspector inspector = colInfo.getObjectInspector(); - if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) { - return toPrimitiveConstDesc(colInfo, inspector); - } - if (inspector instanceof ConstantObjectInspector && inspector instanceof ListObjectInspector) { - ObjectInspector listElementOI = ((ListObjectInspector)inspector).getListElementObjectInspector(); - if (listElementOI instanceof PrimitiveObjectInspector) { - return toListConstDesc(colInfo, inspector, listElementOI); - } - } - if (inspector instanceof ConstantObjectInspector && inspector instanceof MapObjectInspector) { - ObjectInspector keyOI = ((MapObjectInspector)inspector).getMapKeyObjectInspector(); - ObjectInspector valueOI = ((MapObjectInspector)inspector).getMapValueObjectInspector(); - if (keyOI instanceof PrimitiveObjectInspector && valueOI instanceof PrimitiveObjectInspector) { - return toMapConstDesc(colInfo, inspector, keyOI, valueOI); - } - } - if (inspector instanceof ConstantObjectInspector && inspector instanceof StructObjectInspector) { - boolean allPrimitive = true; - List fields = ((StructObjectInspector)inspector).getAllStructFieldRefs(); - for (StructField field : fields) { - allPrimitive &= field.getFieldObjectInspector() instanceof PrimitiveObjectInspector; - } - if (allPrimitive) { - return toStructConstDesc(colInfo, inspector, fields); - } - } - // non-constant or non-primitive constants - ExprNodeColumnDesc column = new ExprNodeColumnDesc(colInfo); - column.setSkewedCol(colInfo.isSkewedCol()); - return column; - } - - private static ExprNodeConstantDesc toPrimitiveConstDesc(ColumnInfo colInfo, ObjectInspector inspector) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector; - Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue(); - ExprNodeConstantDesc constantExpr = - new ExprNodeConstantDesc(colInfo.getType(), poi.getPrimitiveJavaObject(constant)); - constantExpr.setFoldedFromCol(colInfo.getInternalName()); - constantExpr.setFoldedFromTab(colInfo.getTabAlias()); - return constantExpr; - } - - private static ExprNodeConstantDesc toListConstDesc(ColumnInfo colInfo, ObjectInspector inspector, - ObjectInspector listElementOI) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector)listElementOI; - List values = (List)((ConstantObjectInspector) inspector).getWritableConstantValue(); - List constant = new ArrayList(); - for (Object o : values) { - constant.add(poi.getPrimitiveJavaObject(o)); - } - - ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); - constantExpr.setFoldedFromCol(colInfo.getInternalName()); - constantExpr.setFoldedFromTab(colInfo.getTabAlias()); - return constantExpr; - } - - private static ExprNodeConstantDesc toMapConstDesc(ColumnInfo colInfo, ObjectInspector inspector, - ObjectInspector keyOI, ObjectInspector valueOI) { - PrimitiveObjectInspector keyPoi = (PrimitiveObjectInspector)keyOI; - PrimitiveObjectInspector valuePoi = (PrimitiveObjectInspector)valueOI; - Map values = (Map)((ConstantObjectInspector) inspector).getWritableConstantValue(); - Map constant = new LinkedHashMap(); - for (Map.Entry e : values.entrySet()) { - constant.put(keyPoi.getPrimitiveJavaObject(e.getKey()), valuePoi.getPrimitiveJavaObject(e.getValue())); - } - - ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); - constantExpr.setFoldedFromCol(colInfo.getInternalName()); - constantExpr.setFoldedFromTab(colInfo.getTabAlias()); - return constantExpr; - } - - private static ExprNodeConstantDesc toStructConstDesc(ColumnInfo colInfo, ObjectInspector inspector, - List fields) { - List values = (List)((ConstantObjectInspector) inspector).getWritableConstantValue(); - List constant = new ArrayList(); - for (int i = 0; i < values.size(); i++) { - Object value = values.get(i); - PrimitiveObjectInspector fieldPoi = (PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector(); - constant.add(fieldPoi.getPrimitiveJavaObject(value)); - } - ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); - constantExpr.setFoldedFromCol(colInfo.getInternalName()); - constantExpr.setFoldedFromTab(colInfo.getTabAlias()); - return constantExpr; - } - - /** - * Factory method to get ColumnExprProcessor. - * - * @return ColumnExprProcessor. - */ - public ColumnExprProcessor getColumnExprProcessor() { - return new ColumnExprProcessor(); - } - - /** - * The default processor for typechecking. - */ - public static class DefaultExprProcessor implements NodeProcessor { - - static HashMap specialUnaryOperatorTextHashMap; - static HashMap conversionFunctionTextHashMap; - static HashSet windowingTokens; - static { - specialUnaryOperatorTextHashMap = new HashMap(); - specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive"); - specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative"); - conversionFunctionTextHashMap = new HashMap(); - conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, - serdeConstants.BOOLEAN_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_TINYINT, - serdeConstants.TINYINT_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_SMALLINT, - serdeConstants.SMALLINT_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_INT, - serdeConstants.INT_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_BIGINT, - serdeConstants.BIGINT_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_FLOAT, - serdeConstants.FLOAT_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_DOUBLE, - serdeConstants.DOUBLE_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, - serdeConstants.STRING_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_CHAR, - serdeConstants.CHAR_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_VARCHAR, - serdeConstants.VARCHAR_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_BINARY, - serdeConstants.BINARY_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, - serdeConstants.DATE_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMP, - serdeConstants.TIMESTAMP_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMPLOCALTZ, - serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_INTERVAL_YEAR_MONTH, - serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_INTERVAL_DAY_TIME, - serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); - conversionFunctionTextHashMap.put(HiveParser.TOK_DECIMAL, - serdeConstants.DECIMAL_TYPE_NAME); - - windowingTokens = new HashSet(); - windowingTokens.add(HiveParser.KW_OVER); - windowingTokens.add(HiveParser.TOK_PARTITIONINGSPEC); - windowingTokens.add(HiveParser.TOK_DISTRIBUTEBY); - windowingTokens.add(HiveParser.TOK_SORTBY); - windowingTokens.add(HiveParser.TOK_CLUSTERBY); - windowingTokens.add(HiveParser.TOK_WINDOWSPEC); - windowingTokens.add(HiveParser.TOK_WINDOWRANGE); - windowingTokens.add(HiveParser.TOK_WINDOWVALUES); - windowingTokens.add(HiveParser.KW_UNBOUNDED); - windowingTokens.add(HiveParser.KW_PRECEDING); - windowingTokens.add(HiveParser.KW_FOLLOWING); - windowingTokens.add(HiveParser.KW_CURRENT); - windowingTokens.add(HiveParser.TOK_TABSORTCOLNAMEASC); - windowingTokens.add(HiveParser.TOK_TABSORTCOLNAMEDESC); - windowingTokens.add(HiveParser.TOK_NULLS_FIRST); - windowingTokens.add(HiveParser.TOK_NULLS_LAST); - } - - protected static boolean isRedundantConversionFunction(ASTNode expr, - boolean isFunction, ArrayList children) { - if (!isFunction) { - return false; - } - // conversion functions take a single parameter - if (children.size() != 1) { - return false; - } - String funcText = conversionFunctionTextHashMap.get(((ASTNode) expr - .getChild(0)).getType()); - // not a conversion function - if (funcText == null) { - return false; - } - // return true when the child type and the conversion target type is the - // same - return ((PrimitiveTypeInfo) children.get(0).getTypeInfo()).getTypeName() - .equalsIgnoreCase(funcText); - } - - public static String getFunctionText(ASTNode expr, boolean isFunction) { - String funcText = null; - if (!isFunction) { - // For operator, the function name is the operator text, unless it's in - // our special dictionary - if (expr.getChildCount() == 1) { - funcText = specialUnaryOperatorTextHashMap.get(expr.getType()); - } - if(funcText == null) { - funcText = expr.getText(); - } - } else { - // For TOK_FUNCTION, the function name is stored in the first child, - // unless it's in our - // special dictionary. - assert (expr.getChildCount() >= 1); - int funcType = ((ASTNode) expr.getChild(0)).getType(); - if (funcText == null) { - funcText = conversionFunctionTextHashMap.get(funcType); - } - if (funcText == null) { - funcText = ((ASTNode) expr.getChild(0)).getText(); - } - } - return BaseSemanticAnalyzer.unescapeIdentifier(funcText); - } - - /** - * This function create an ExprNodeDesc for a UDF function given the - * children (arguments). It will insert implicit type conversion functions - * if necessary. - * - * @throws UDFArgumentException - */ - static ExprNodeDesc getFuncExprNodeDescWithUdfData(String udfName, TypeInfo typeInfo, - ExprNodeDesc... children) throws UDFArgumentException { - - FunctionInfo fi; - try { - fi = FunctionRegistry.getFunctionInfo(udfName); - } catch (SemanticException e) { - throw new UDFArgumentException(e); - } - if (fi == null) { - throw new UDFArgumentException(udfName + " not found."); - } - - GenericUDF genericUDF = fi.getGenericUDF(); - if (genericUDF == null) { - throw new UDFArgumentException(udfName - + " is an aggregation function or a table function."); - } - - // Add udfData to UDF if necessary - if (typeInfo != null) { - if (genericUDF instanceof SettableUDF) { - ((SettableUDF)genericUDF).setTypeInfo(typeInfo); - } - } - - List childrenList = new ArrayList(children.length); - - childrenList.addAll(Arrays.asList(children)); - return ExprNodeGenericFuncDesc.newInstance(genericUDF, - childrenList); - } - - public static ExprNodeDesc getFuncExprNodeDesc(String udfName, - ExprNodeDesc... children) throws UDFArgumentException { - return getFuncExprNodeDescWithUdfData(udfName, null, children); - } - - protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, FunctionInfo fi, - List children, GenericUDF genericUDF) throws SemanticException { - // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't - // supported - if (fi.getGenericUDTF() != null) { - throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg()); - } - // UDAF in filter condition, group-by caluse, param of funtion, etc. - if (fi.getGenericUDAFResolver() != null) { - if (isFunction) { - throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg((ASTNode) expr - .getChild(0))); - } else { - throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr)); - } - } - if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) { - if (FunctionRegistry.isStateful(genericUDF)) { - throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg()); - } - } - } - - protected void insertCast(String funcText, ArrayList children) throws SemanticException { - // substring, concat UDFs expect first argument as string. Therefore this method inserts explicit cast - // to cast the first operand to string - if (funcText.equals("substring") || funcText.equals("concat")){ - if(children.size() > 0 && !ExprNodeDescUtils.isStringType(children.get(0))) { - ExprNodeDesc newColumn = ParseUtils.createConversionCast(children.get(0), TypeInfoFactory.stringTypeInfo); - children.set(0, newColumn); - } - } - } - - protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, - boolean isFunction, ArrayList children, TypeCheckCtx ctx) - throws SemanticException, UDFArgumentException { - // return the child directly if the conversion is redundant. - if (isRedundantConversionFunction(expr, isFunction, children)) { - assert (children.size() == 1); - assert (children.get(0) != null); - return children.get(0); - } - String funcText = getFunctionText(expr, isFunction); - ExprNodeDesc desc; - if (funcText.equals(".")) { - // "." : FIELD Expression - - assert (children.size() == 2); - // Only allow constant field name for now - assert (children.get(1) instanceof ExprNodeConstantDesc); - ExprNodeDesc object = children.get(0); - ExprNodeConstantDesc fieldName = (ExprNodeConstantDesc) children.get(1); - assert (fieldName.getValue() instanceof String); - - // Calculate result TypeInfo - String fieldNameString = (String) fieldName.getValue(); - TypeInfo objectTypeInfo = object.getTypeInfo(); - - // Allow accessing a field of list element structs directly from a list - boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); - if (isList) { - objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); - } - if (objectTypeInfo.getCategory() != Category.STRUCT) { - throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); - } - TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); - if (isList) { - t = TypeInfoFactory.getListTypeInfo(t); - } - - desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString, isList); - } else if (funcText.equals("[")) { - // "[]" : LSQUARE/INDEX Expression - if (!ctx.getallowIndexExpr()) { - throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr)); - } - - assert (children.size() == 2); - - // Check whether this is a list or a map - TypeInfo myt = children.get(0).getTypeInfo(); - - if (myt.getCategory() == Category.LIST) { - // Only allow integer index for now - if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), - TypeInfoFactory.intTypeInfo)) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage( - expr, ErrorMsg.INVALID_ARRAYINDEX_TYPE.getMsg())); - } - - // Calculate TypeInfo - TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); - desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); - } else if (myt.getCategory() == Category.MAP) { - if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), - ((MapTypeInfo) myt).getMapKeyTypeInfo())) { - throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE - .getMsg(expr)); - } - // Calculate TypeInfo - TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); - desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); - } else { - throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, myt.getTypeName())); - } - } else { - // other operators or functions - FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcText); - - if (fi == null) { - if (isFunction) { - throw new SemanticException(ErrorMsg.INVALID_FUNCTION - .getMsg((ASTNode) expr.getChild(0))); - } else { - throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr)); - } - } - - // getGenericUDF() actually clones the UDF. Just call it once and reuse. - GenericUDF genericUDF = fi.getGenericUDF(); - - if (!fi.isNative()) { - ctx.getUnparseTranslator().addIdentifierTranslation( - (ASTNode) expr.getChild(0)); - } - - // Handle type casts that may contain type parameters - if (isFunction) { - ASTNode funcNameNode = (ASTNode)expr.getChild(0); - switch (funcNameNode.getType()) { - case HiveParser.TOK_CHAR: - // Add type params - CharTypeInfo charTypeInfo = ParseUtils.getCharTypeInfo(funcNameNode); - if (genericUDF != null) { - ((SettableUDF)genericUDF).setTypeInfo(charTypeInfo); - } - break; - case HiveParser.TOK_VARCHAR: - VarcharTypeInfo varcharTypeInfo = ParseUtils.getVarcharTypeInfo(funcNameNode); - if (genericUDF != null) { - ((SettableUDF)genericUDF).setTypeInfo(varcharTypeInfo); - } - break; - case HiveParser.TOK_TIMESTAMPLOCALTZ: - TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = new TimestampLocalTZTypeInfo(); - HiveConf conf; - try { - conf = Hive.get().getConf(); - } catch (HiveException e) { - throw new SemanticException(e); - } - timestampLocalTZTypeInfo.setTimeZone(conf.getLocalTimeZone()); - if (genericUDF != null) { - ((SettableUDF)genericUDF).setTypeInfo(timestampLocalTZTypeInfo); - } - break; - case HiveParser.TOK_DECIMAL: - DecimalTypeInfo decTypeInfo = ParseUtils.getDecimalTypeTypeInfo(funcNameNode); - if (genericUDF != null) { - ((SettableUDF)genericUDF).setTypeInfo(decTypeInfo); - } - break; - default: - // Do nothing - break; - } - } - - insertCast(funcText, children); - - validateUDF(expr, isFunction, ctx, fi, children, genericUDF); - - // Try to infer the type of the constant only if there are two - // nodes, one of them is column and the other is numeric const - if (genericUDF instanceof GenericUDFBaseCompare - && children.size() == 2 - && ((children.get(0) instanceof ExprNodeConstantDesc - && children.get(1) instanceof ExprNodeColumnDesc) - || (children.get(0) instanceof ExprNodeColumnDesc - && children.get(1) instanceof ExprNodeConstantDesc))) { - - int constIdx = children.get(0) instanceof ExprNodeConstantDesc ? 0 : 1; - - ExprNodeDesc constChild = children.get(constIdx); - ExprNodeDesc columnChild = children.get(1 - constIdx); - - final PrimitiveTypeInfo colTypeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase()); - ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild); - if (newChild == null) { - // non-interpretable as target type... - // TODO: all comparisons with null should result in null - if (genericUDF instanceof GenericUDFOPEqual - && !(genericUDF instanceof GenericUDFOPEqualNS)) { - return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null); - } - } else { - children.set(constIdx, newChild); - } - } - if (genericUDF instanceof GenericUDFIn) { - - ExprNodeDesc columnDesc = children.get(0); - List outputOpList = children.subList(1, children.size()); - ArrayList inOperands = new ArrayList<>(outputOpList); - outputOpList.clear(); - - boolean hasNullValue = false; - for (ExprNodeDesc oldChild : inOperands) { - if (oldChild == null) { - hasNullValue = true; - continue; - } - ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, oldChild); - if (newChild == null) { - hasNullValue = true; - continue; - } - outputOpList.add(newChild); - } - - if (hasNullValue) { - ExprNodeConstantDesc nullConst = new ExprNodeConstantDesc(columnDesc.getTypeInfo(), null); - if (outputOpList.size() == 0) { - // we have found only null values...remove the IN ; it will be null all the time. - return nullConst; - } - outputOpList.add(nullConst); - } - - if (!ctx.isCBOExecuted()) { - - HiveConf conf; - try { - conf = Hive.get().getConf(); - } catch (HiveException e) { - throw new SemanticException(e); - } - if(children.size() <= HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES)) { - ArrayList orOperands = TypeCheckProcFactoryUtils.rewriteInToOR(children); - if (orOperands != null) { - if (orOperands.size() == 1) { - orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false)); - } - funcText = "or"; - genericUDF = new GenericUDFOPOr(); - children.clear(); - children.addAll(orOperands); - } - } - } - } - if (genericUDF instanceof GenericUDFOPOr) { - // flatten OR - List childrenList = new ArrayList( - children.size()); - for (ExprNodeDesc child : children) { - if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) { - child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean")); - } - if (FunctionRegistry.isOpOr(child)) { - childrenList.addAll(child.getChildren()); - } else { - childrenList.add(child); - } - } - desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, - childrenList); - } else if (genericUDF instanceof GenericUDFOPAnd) { - // flatten AND - List childrenList = new ArrayList( - children.size()); - for (ExprNodeDesc child : children) { - if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) { - child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean")); - } - if (FunctionRegistry.isOpAnd(child)) { - childrenList.addAll(child.getChildren()); - } else { - childrenList.add(child); - } - } - desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, - childrenList); - } else if (ctx.isFoldExpr() && canConvertIntoCoalesce(genericUDF, children)) { - // Rewrite CASE into COALESCE - desc = ExprNodeGenericFuncDesc.newInstance(new GenericUDFCoalesce(), - Lists.newArrayList(children.get(0), new ExprNodeConstantDesc(false))); - if (Boolean.FALSE.equals(((ExprNodeConstantDesc) children.get(1)).getValue())) { - desc = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), - Lists.newArrayList(desc)); - } - } else { - desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, - children); - } - - // If the function is deterministic and the children are constants, - // we try to fold the expression to remove e.g. cast on constant - if (ctx.isFoldExpr() && desc instanceof ExprNodeGenericFuncDesc && - FunctionRegistry.isConsistentWithinQuery(genericUDF) && - ExprNodeDescUtils.isAllConstants(children)) { - ExprNodeDesc constantExpr = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)desc); - if (constantExpr != null) { - desc = constantExpr; - } - } - } - // UDFOPPositive is a no-op. - // However, we still create it, and then remove it here, to make sure we - // only allow - // "+" for numeric types. - if (FunctionRegistry.isOpPositive(desc)) { - assert (desc.getChildren().size() == 1); - desc = desc.getChildren().get(0); - } - assert (desc != null); - return desc; - } - - /** - * Interprets the given value as columnDesc if possible - */ - private static ExprNodeDesc interpretNodeAsStruct(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) - throws SemanticException { - if(columnDesc instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc exprNodeColumnDesc = (ExprNodeColumnDesc) columnDesc; - final PrimitiveTypeInfo typeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(exprNodeColumnDesc.getTypeString().toLowerCase()); - return interpretNodeAs(typeInfo, valueDesc); - } - if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isConstantStruct(valueDesc)) { - List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); - ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; - StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); - ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); - ArrayList newStructFieldInfos = new ArrayList<>(); - - if (columnChilds.size() != structFieldInfos.size()) { - throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos)); - } - List oldValues = (List) valueConstDesc.getValue(); - List newValues = new ArrayList<>(); - for (int i = 0; i < columnChilds.size(); i++) { - newStructFieldInfos.add(columnChilds.get(i).getTypeInfo()); - Object newValue = interpretConstantAsPrimitive( - (PrimitiveTypeInfo) columnChilds.get(i).getTypeInfo(), - oldValues.get(i), - structFieldInfos.get(i)); - newValues.add(newValue); - } - StructTypeInfo sti = new StructTypeInfo(); - sti.setAllStructFieldTypeInfos(newStructFieldInfos); - sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); - return new ExprNodeConstantDesc(sti, newValues); - - } - if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isStructUDF(valueDesc)) { - List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); - List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren(); - if (columnChilds.size() != valueChilds.size()) { - throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds)); - } - List oldValueChilds = new ArrayList<>(valueChilds); - valueChilds.clear(); - for (int i = 0; i < oldValueChilds.size(); i++) { - ExprNodeDesc newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i)); - valueChilds.add(newValue); - } - } - return valueDesc; - } - - @VisibleForTesting - protected static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) { - if (constChild instanceof ExprNodeConstantDesc) { - // Try to narrow type of constant - Object constVal = ((ExprNodeConstantDesc) constChild).getValue(); - if (constVal == null) { - // adjust type of null - return new ExprNodeConstantDesc(colTypeInfo, null); - } - Object newConst = interpretConstantAsPrimitive(colTypeInfo, constVal, constChild.getTypeInfo()); - if (newConst == null) { - return null; - } - if(newConst == constVal) { - return constChild; - } else { - return new ExprNodeConstantDesc(adjustType(colTypeInfo, newConst), newConst); - } - } - return constChild; - } - - private static TypeInfo adjustType(PrimitiveTypeInfo colTypeInfo, Object newConst) { - if (newConst instanceof HiveDecimal) { - return NumExprProcessor.adjustType((HiveDecimal) newConst); - } - return colTypeInfo; - } - - private static BigDecimal toBigDecimal(String val) { - if (!NumberUtils.isNumber(val)) { - throw new NumberFormatException("The given string is not a valid number: " + val); - } - return new BigDecimal(val.replaceAll("[dDfFlL]$", "")); - } - - private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo colTypeInfo, Object constVal, - TypeInfo constTypeInfo) { - if (constVal instanceof Number || constVal instanceof String) { - try { - PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); - if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { - return toBigDecimal(constVal.toString()).intValueExact(); - } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { - return toBigDecimal(constVal.toString()).longValueExact(); - } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { - return Double.valueOf(constVal.toString()); - } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { - return Float.valueOf(constVal.toString()); - } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { - return toBigDecimal(constVal.toString()).byteValueExact(); - } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { - return toBigDecimal(constVal.toString()).shortValueExact(); - } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { - return HiveDecimal.create(constVal.toString()); - } - } catch (NumberFormatException | ArithmeticException nfe) { - LOG.trace("Failed to narrow type of constant", nfe); - return null; - } - } - - // Comparision of decimal and float/double happens in float/double. - if (constVal instanceof HiveDecimal) { - HiveDecimal hiveDecimal = (HiveDecimal) constVal; - - PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); - if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { - return hiveDecimal.doubleValue(); - } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { - return hiveDecimal.floatValue(); - } - return hiveDecimal; - } - - String constTypeInfoName = constTypeInfo.getTypeName(); - if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) { - // because a comparison against a "string" will happen in "string" type. - // to avoid unintnetional comparisions in "string" - // constants which are representing char/varchar values must be converted to the - // appropriate type. - if (colTypeInfo instanceof CharTypeInfo) { - final String constValue = constVal.toString(); - final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); - HiveChar newValue = new HiveChar(constValue, length); - HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH); - if (maxCharConst.equals(newValue)) { - return newValue; - } else { - return null; - } - } - if (colTypeInfo instanceof VarcharTypeInfo) { - final String constValue = constVal.toString(); - final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); - HiveVarchar newValue = new HiveVarchar(constValue, length); - HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH); - if (maxCharConst.equals(newValue)) { - return newValue; - } else { - return null; - } - } - } - - return constVal; - } - - private boolean canConvertIntoCoalesce(GenericUDF genericUDF, ArrayList children) { - if (genericUDF instanceof GenericUDFWhen && children.size() == 3 && - children.get(1) instanceof ExprNodeConstantDesc && - children.get(2) instanceof ExprNodeConstantDesc) { - ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) children.get(1); - ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) children.get(2); - Object thenVal = constThen.getValue(); - Object elseVal = constElse.getValue(); - if (thenVal instanceof Boolean && elseVal instanceof Boolean) { - return true; - } - } - return false; - } - - /** - * Returns true if des is a descendant of ans (ancestor) - */ - private boolean isDescendant(Node ans, Node des) { - if (ans.getChildren() == null) { - return false; - } - for (Node c : ans.getChildren()) { - if (c == des) { - return true; - } - if (isDescendant(c, des)) { - return true; - } - } - return false; - } - - protected ExprNodeDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr, - Object... nodeOutputs) throws SemanticException { - RowResolver input = ctx.getInputRR(); - String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0) - .getText()); - // NOTE: tableAlias must be a valid non-ambiguous table alias, - // because we've checked that in TOK_TABLE_OR_COL's process method. - String colName; - if (nodeOutputs[1] instanceof ExprNodeConstantDesc) { - colName = ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString(); - } else if (nodeOutputs[1] instanceof ExprNodeColumnDesc) { - colName = ((ExprNodeColumnDesc)nodeOutputs[1]).getColumn(); - } else { - throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]); - } - ColumnInfo colInfo = input.get(tableAlias, colName); - - // Try outer Row resolver - if(colInfo == null && ctx.getOuterRR() != null) { - RowResolver outerRR = ctx.getOuterRR(); - colInfo = outerRR.get(tableAlias, colName); - } - - if (colInfo == null) { - ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr); - return null; - } - return toExprNodeDesc(colInfo); - } - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - // Here we know nd represents a group by expression. - - // During the DFS traversal of the AST, a descendant of nd likely set an - // error because a sub-tree of nd is unlikely to also be a group by - // expression. For example, in a query such as - // SELECT *concat(key)* FROM src GROUP BY concat(key), 'key' will be - // processed before 'concat(key)' and since 'key' is not a group by - // expression, an error will be set in ctx by ColumnExprProcessor. - - // We can clear the global error when we see that it was set in a - // descendant node of a group by expression because - // processGByExpr() returns a ExprNodeDesc that effectively ignores - // its children. Although the error can be set multiple times by - // descendant nodes, DFS traversal ensures that the error only needs to - // be cleared once. Also, for a case like - // SELECT concat(value, concat(value))... the logic still works as the - // error is only set with the first 'value'; all node processors quit - // early if the global error is set. - - if (isDescendant(nd, ctx.getErrorSrcNode())) { - ctx.setError(null, null); - } - return desc; - } - - if (ctx.getError() != null) { - return null; - } - - ASTNode expr = (ASTNode) nd; - - /* - * A Windowing specification get added as a child to a UDAF invocation to distinguish it - * from similar UDAFs but on different windows. - * The UDAF is translated to a WindowFunction invocation in the PTFTranslator. - * So here we just return null for tokens that appear in a Window Specification. - * When the traversal reaches up to the UDAF invocation its ExprNodeDesc is build using the - * ColumnInfo in the InputRR. This is similar to how UDAFs are handled in Select lists. - * The difference is that there is translation for Window related tokens, so we just - * return null; - */ - if (windowingTokens.contains(expr.getType())) { - if (!ctx.getallowWindowing()) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); - } - - return null; - } - - if(expr.getType() == HiveParser.TOK_SUBQUERY_OP || expr.getType() == HiveParser.TOK_QUERY) { - return null; - } - - if (expr.getType() == HiveParser.TOK_TABNAME) { - return null; - } - - if (expr.getType() == HiveParser.TOK_ALLCOLREF) { - if (!ctx.getallowAllColRef()) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN - .getMsg("All column reference is not supported in the context"))); - } - - RowResolver input = ctx.getInputRR(); - ExprNodeColumnListDesc columnList = new ExprNodeColumnListDesc(); - assert expr.getChildCount() <= 1; - if (expr.getChildCount() == 1) { - // table aliased (select a.*, for example) - ASTNode child = (ASTNode) expr.getChild(0); - assert child.getType() == HiveParser.TOK_TABNAME; - assert child.getChildCount() == 1; - String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText()); - Map columns = input.getFieldMap(tableAlias); - if (columns == null) { - throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(child)); - } - for (Map.Entry colMap : columns.entrySet()) { - ColumnInfo colInfo = colMap.getValue(); - if (!colInfo.getIsVirtualCol()) { - columnList.addColumn(toExprNodeDesc(colInfo)); - } - } - } else { - // all columns (select *, for example) - for (ColumnInfo colInfo : input.getColumnInfos()) { - if (!colInfo.getIsVirtualCol()) { - columnList.addColumn(toExprNodeDesc(colInfo)); - } - } - } - return columnList; - } - - // If the first child is a TOK_TABLE_OR_COL, and nodeOutput[0] is NULL, - // and the operator is a DOT, then it's a table column reference. - if (expr.getType() == HiveParser.DOT - && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL - && nodeOutputs[0] == null) { - return processQualifiedColRef(ctx, expr, nodeOutputs); - } - - // Return nulls for conversion operators - if (conversionFunctionTextHashMap.keySet().contains(expr.getType()) - || expr.getToken().getType() == HiveParser.CharSetName - || expr.getToken().getType() == HiveParser.CharSetLiteral) { - return null; - } - - boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION || - expr.getType() == HiveParser.TOK_FUNCTIONSTAR || - expr.getType() == HiveParser.TOK_FUNCTIONDI); - - if (!ctx.getAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) { - throw new SemanticException( - SemanticAnalyzer.generateErrorMessage(expr, ErrorMsg.DISTINCT_NOT_SUPPORTED.getMsg())); - } - - // Create all children - int childrenBegin = (isFunction ? 1 : 0); - ArrayList children = new ArrayList( - expr.getChildCount() - childrenBegin); - for (int ci = childrenBegin; ci < expr.getChildCount(); ci++) { - if (nodeOutputs[ci] instanceof ExprNodeColumnListDesc) { - children.addAll(((ExprNodeColumnListDesc) nodeOutputs[ci]).getChildren()); - } else { - children.add((ExprNodeDesc) nodeOutputs[ci]); - } - } - - if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) { - if (!ctx.getallowFunctionStar()) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, - ErrorMsg.INVALID_COLUMN - .getMsg(".* reference is not supported in the context"))); - } - - RowResolver input = ctx.getInputRR(); - for (ColumnInfo colInfo : input.getColumnInfos()) { - if (!colInfo.getIsVirtualCol()) { - children.add(toExprNodeDesc(colInfo)); - } - } - } - - // If any of the children contains null, then return a null - // this is a hack for now to handle the group by case - if (children.contains(null)) { - List possibleColumnNames = getReferenceableColumnAliases(ctx); - String reason = String.format("(possible column names are: %s)", - StringUtils.join(possibleColumnNames, ", ")); - ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(0), reason), - expr); - return null; - } - - // Create function desc - try { - return getXpathOrFuncExprNodeDesc(expr, isFunction, children, ctx); - } catch (UDFArgumentTypeException e) { - throw new SemanticException(ErrorMsg.INVALID_ARGUMENT_TYPE.getMsg(expr - .getChild(childrenBegin + e.getArgumentId()), e.getMessage()), e); - } catch (UDFArgumentLengthException e) { - throw new SemanticException(ErrorMsg.INVALID_ARGUMENT_LENGTH.getMsg( - expr, e.getMessage()), e); - } catch (UDFArgumentException e) { - throw new SemanticException(ErrorMsg.INVALID_ARGUMENT.getMsg(expr, e - .getMessage()), e); - } - } - - protected List getReferenceableColumnAliases(TypeCheckCtx ctx) { - return ctx.getInputRR().getReferenceableColumnAliases(null, -1); - } - } - - /** - * Factory method to get DefaultExprProcessor. - * - * @return DefaultExprProcessor. - */ - public DefaultExprProcessor getDefaultExprProcessor() { - return new DefaultExprProcessor(); - } - - /** - * Processor for subquery expressions.. - */ - public static class SubQueryExprProcessor implements NodeProcessor { - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TypeCheckCtx ctx = (TypeCheckCtx) procCtx; - if (ctx.getError() != null) { - return null; - } - - ASTNode expr = (ASTNode) nd; - ASTNode sqNode = (ASTNode) expr.getParent().getChild(1); - - if (!ctx.getallowSubQueryExpr()) { - throw new CalciteSubquerySemanticException(SemanticAnalyzer.generateErrorMessage(sqNode, - ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg("Currently SubQuery expressions are only allowed as " + - "Where and Having Clause predicates"))); - } - - ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx); - if (desc != null) { - return desc; - } - - //TOK_SUBQUERY_EXPR should have either 2 or 3 children - assert(expr.getChildren().size() == 3 || expr.getChildren().size() == 2); - //First child should be operand - assert(expr.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP); - - ASTNode subqueryOp = (ASTNode) expr.getChild(0); - - boolean isIN = (subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_IN - || subqueryOp.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP_NOTIN); - boolean isEXISTS = (subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_EXISTS - || subqueryOp.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP_NOTEXISTS); - boolean isSOME = (subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_SOME); - boolean isALL = (subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_ALL); - boolean isScalar = subqueryOp.getChildCount() == 0; - - // subqueryToRelNode might be null if subquery expression anywhere other than - // as expected in filter (where/having). We should throw an appropriate error - // message - - Map subqueryToRelNode = ctx.getSubqueryToRelNode(); - if(subqueryToRelNode == null) { - throw new CalciteSubquerySemanticException(ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg( - " Currently SubQuery expressions are only allowed as " + - "Where and Having Clause predicates")); - } - - RelNode subqueryRel = subqueryToRelNode.get(expr); - - //For now because subquery is only supported in filter - // we will create subquery expression of boolean type - if(isEXISTS) { - return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, - ExprNodeSubQueryDesc.SubqueryType.EXISTS); - } else if(isIN) { - assert(nodeOutputs[2] != null); - ExprNodeDesc lhs = (ExprNodeDesc)nodeOutputs[2]; - return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, - ExprNodeSubQueryDesc.SubqueryType.IN, lhs); - } else if(isScalar){ - // only single subquery expr is supported - if(subqueryRel.getRowType().getFieldCount() != 1) { - throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg( - "More than one column expression in subquery")); - } - // figure out subquery expression column's type - TypeInfo subExprType = TypeConverter.convert(subqueryRel.getRowType().getFieldList().get(0).getType()); - return new ExprNodeSubQueryDesc(subExprType, subqueryRel, - ExprNodeSubQueryDesc.SubqueryType.SCALAR); - } else if(isSOME) { - assert(nodeOutputs[2] != null); - ExprNodeDesc lhs = (ExprNodeDesc)nodeOutputs[2]; - return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, - ExprNodeSubQueryDesc.SubqueryType.SOME, lhs, (ASTNode)subqueryOp.getChild(1)); - } else if(isALL) { - assert(nodeOutputs[2] != null); - ExprNodeDesc lhs = (ExprNodeDesc)nodeOutputs[2]; - return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, - ExprNodeSubQueryDesc.SubqueryType.ALL, lhs, (ASTNode)subqueryOp.getChild(1)); - } - - /* - * Restriction.1.h :: SubQueries only supported in the SQL Where Clause. - */ - ctx.setError(ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg(sqNode, - "Currently only IN & EXISTS SubQuery expressions are allowed"), - sqNode); - return null; - } - } - - /** - * Factory method to get SubQueryExprProcessor. - * - * @return DateExprProcessor. - */ - public SubQueryExprProcessor getSubQueryExprProcessor() { - return new SubQueryExprProcessor(); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java index a8a83c8691..02fc69dfc4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java @@ -175,7 +175,7 @@ void addTableNameTranslation(ASTNode tableName, String currentDatabaseName) { * @param node * source node (which must be an identifier) to be replaced */ - void addIdentifierTranslation(ASTNode identifier) { + public void addIdentifierTranslation(ASTNode identifier) { if (!enabled) { return; } @@ -198,7 +198,7 @@ void addIdentifierTranslation(ASTNode identifier) { * @param sourceNode the node providing the replacement text * */ - void addCopyTranslation(ASTNode targetNode, ASTNode sourceNode) { + public void addCopyTranslation(ASTNode targetNode, ASTNode sourceNode) { if (!enabled) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprFactory.java new file mode 100644 index 0000000000..0023522cd1 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprFactory.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.type; + +import java.math.BigDecimal; +import java.time.ZoneId; +import java.util.List; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.SubqueryType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hive.common.util.DateUtils; + +/** + * Generic expressions factory. Currently, the only implementation produces + * Hive {@link ExprNodeDesc}. + */ +public abstract class ExprFactory { + + static final BigDecimal NANOS_PER_SEC_BD = + new BigDecimal(DateUtils.NANOS_PER_SEC); + + /** + * Returns whether the input is an instance of the expression class. + */ + protected abstract boolean isExprInstance(Object value); + + /** + * Generates an expression from the input column. This may not necessarily + * be a column expression, e.g., if the column is a constant. + */ + protected abstract T toExpr(ColumnInfo colInfo); + + /* FIELD REFERENCES */ + /** + * Returns whether the input expression is a column reference expression. + */ + protected abstract boolean isColumnRefExpr(T expr); + + /** + * Creates column expression. + */ + protected abstract T createColumnRefExpr(ColumnInfo colInfo); + + /** + * Returns column name referenced by a column expression. + */ + protected abstract String getColumnName(T expr); + + /* CONSTANT EXPRESSIONS */ + /** + * Returns whether the input expression is a constant expression. + */ + protected abstract boolean isConstantExpr(T expr); + + /** + * Returns whether all input expressions are constant expressions. + */ + protected boolean isAllConstants(List exprs) { + for (T expr : exprs) { + if (!isConstantExpr(expr)) { + return false; + } + } + return true; + } + + /** + * Returns whether the input expression is a struct of + * constant expressions (all of them). + */ + protected abstract boolean isConstantStruct(T expr); + + /** + * Creates a null constant expression with void type. + */ + protected abstract T createNullConstantExpr(); + + /** + * Creates a boolean constant expression from input value. + */ + protected abstract T createBooleanConstantExpr(String value); + + /** + * Creates a bigint constant expression from input value. + */ + protected abstract T createBigintConstantExpr(String value); + + /** + * Creates a int constant expression from input value. + */ + protected abstract T createIntConstantExpr(String value); + + /** + * Creates a smallint constant expression from input value. + */ + protected abstract T createSmallintConstantExpr(String value); + + /** + * Creates a tinyint constant expression from input value. + */ + protected abstract T createTinyintConstantExpr(String value); + + /** + * Creates a float constant expression from input value. + */ + protected abstract T createFloatConstantExpr(String value); + + /** + * Creates a double constant expression from input value. + */ + protected abstract T createDoubleConstantExpr(String value); + + /** + * Creates a decimal constant expression from input value. + * If the constant created from the input value is null, we return: + * 1) a constant expression containing null value if allowNullValueConstantExpr is true, or + * 2) null if allowNullValueConstantExpr is false. + */ + protected abstract T createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr); + + /** + * Creates a string constant expression from input value. + */ + protected abstract T createStringConstantExpr(String value); + + /** + * Creates a date constant expression from input value. + */ + protected abstract T createDateConstantExpr(String value); + + /** + * Creates a timestamp constant expression from input value. + */ + protected abstract T createTimestampConstantExpr(String value); + + /** + * Creates a timestamp with local time zone constant expression from input value. + * ZoneId is the local time zone. + */ + protected abstract T createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId); + + /** + * Creates a interval year-month constant expression from input value. + */ + protected abstract T createIntervalYearMonthConstantExpr(String value); + + /** + * Creates a interval day-time constant expression from input value. + */ + protected abstract T createIntervalDayTimeConstantExpr(String value); + + /** + * Creates a interval year constant expression from input value. + */ + protected abstract T createIntervalYearConstantExpr(String value); + + /** + * Creates a interval month constant expression from input value. + */ + protected abstract T createIntervalMonthConstantExpr(String value); + + /** + * Creates a interval day constant expression from input value. + */ + protected abstract T createIntervalDayConstantExpr(String value); + + /** + * Creates a interval hour constant expression from input value. + */ + protected abstract T createIntervalHourConstantExpr(String value); + + /** + * Creates a interval minute constant expression from input value. + */ + protected abstract T createIntervalMinuteConstantExpr(String value); + + /** + * Creates a interval second constant expression from input value. + */ + protected abstract T createIntervalSecondConstantExpr(String value); + + /** + * Default generator for constant expression when type cannot be inferred + * from input query. + */ + protected T createConstantExpr(String value) { + // The expression can be any one of Double, Long and Integer. We + // try to parse the expression in that order to ensure that the + // most specific type is used for conversion. + T result = null; + try { + result = createDoubleConstantExpr(value); + if (value != null && !value.toLowerCase().contains("e")) { + result = createDecimalConstantExpr(value, false); + if (result != null) { + value = null; // We will use decimal if all else fails. + } + } + result = createBigintConstantExpr(value); + result = createIntConstantExpr(value); + } catch (NumberFormatException e) { + // do nothing here, we will throw an exception in the following block + } + return result; + } + + /** + * Creates a constant expression from input value with given type. + */ + protected abstract T createConstantExpr(TypeInfo typeInfo, Object constantValue); + + /** + * Adjust type of constant value based on input type, e.g., adjust precision and scale + * of decimal value based on type information. + */ + protected abstract TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue); + + /** + * Interpret the input constant value of source type as target type. + */ + protected abstract Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, + Object constantValue, PrimitiveTypeInfo sourceType); + + /** + * Returns value stored in a constant expression. + */ + protected abstract Object getConstantValue(T expr); + + /* METHODS FOR NESTED FIELD REFERENCES CREATION */ + /** + * Creates a reference to a nested field. + */ + protected abstract T createNestedColumnRefExpr( + TypeInfo typeInfo, T expr, String fieldName, Boolean isList); + + /* FUNCTIONS */ + /** + * Returns whether the input expression is a function call. + */ + protected abstract boolean isFuncCallExpr(T expr); + + /** + * Creates function call expression. + */ + protected abstract T createFuncCallExpr(TypeInfo typeInfo, GenericUDF genericUDF, + List inputs); + + /** + * Creates function call expression. + */ + protected abstract T createFuncCallExpr(GenericUDF genericUDF, String funcText, + List inputs) throws UDFArgumentException; + + /** + * Returns whether the input expression is an OR function call. + */ + protected abstract boolean isORFuncCallExpr(T expr); + + /** + * Returns whether the input expression is an AND function call. + */ + protected abstract boolean isANDFuncCallExpr(T expr); + + /** + * Returns whether the input expression is a POSITIVE function call. + */ + protected abstract boolean isPOSITIVEFuncCallExpr(T expr); + + /** + * Returns whether the input expression is a STRUCT function call. + */ + protected abstract boolean isSTRUCTFuncCallExpr(T expr); + + /** + * The method tries to rewrite an IN function call into an OR/AND function call. + * Returns null if the transformation fails. + */ + protected abstract List rewriteINIntoORFuncCallExpr(List inOperands) throws SemanticException; + + /** + * Returns true if a CASE expression can be converted into a COALESCE function call. + */ + protected abstract boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List inputs); + + /* SUBQUERIES */ + /** + * Creates subquery expression. + */ + protected abstract T createSubqueryExpr(TypeCheckCtx ctx, ASTNode subqueryOp, SubqueryType subqueryType, + T[] inputs) throws CalciteSubquerySemanticException; + + /* LIST OF EXPRESSIONS */ + /** + * Returns whether the input expression is a list of expressions. + */ + protected abstract boolean isExprsListExpr(T expr); + + /** + * Creates list of expressions. + */ + protected abstract T createExprsListExpr(); + + /** + * Adds expression to list of expressions and returns resulting + * list. + * If column list is mutable, it will not create a copy + * of the input list. + */ + protected abstract T addExprToExprsList(T columnList, T expr); + + /* TYPE SYSTEM */ + /** + * Returns the type for the input expression. + */ + protected abstract TypeInfo getTypeInfo(T expr); + + /** + * Changes the type of the input expression to the input type and + * returns resulting expression. + * If the input expression is mutable, it will not create a copy + * of the expression. + */ + protected abstract T setTypeInfo(T expr, TypeInfo type); + + /* MISC */ + /** + * Folds the input expression and returns resulting expression. + * If the input expression is mutable, it will not create a copy + * of the expression. + */ + protected abstract T foldExpr(T expr); + + /** + * Returns the children from the input expression (if any). + */ + protected abstract List getExprChildren(T expr); + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeDescExprFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeDescExprFactory.java new file mode 100644 index 0000000000..6be676fd91 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeDescExprFactory.java @@ -0,0 +1,759 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.type; + +import java.math.BigDecimal; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampTZ; +import org.apache.hadoop.hive.common.type.TimestampTZUtil; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeSubQueryDesc; +import org.apache.hadoop.hive.ql.plan.SubqueryType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.apache.hadoop.io.NullWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Expression factory for Hive {@link ExprNodeDesc}. + */ +public class ExprNodeDescExprFactory extends ExprFactory { + + private static final Logger LOG = LoggerFactory.getLogger(ExprNodeDescExprFactory.class); + + /** + * {@inheritDoc} + */ + @Override + protected boolean isExprInstance(Object value) { + return value instanceof ExprNodeDesc; + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeDesc toExpr(ColumnInfo colInfo) { + ObjectInspector inspector = colInfo.getObjectInspector(); + if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) { + return toPrimitiveConstDesc(colInfo, inspector); + } + if (inspector instanceof ConstantObjectInspector && inspector instanceof ListObjectInspector) { + ObjectInspector listElementOI = ((ListObjectInspector)inspector).getListElementObjectInspector(); + if (listElementOI instanceof PrimitiveObjectInspector) { + return toListConstDesc(colInfo, inspector, listElementOI); + } + } + if (inspector instanceof ConstantObjectInspector && inspector instanceof MapObjectInspector) { + ObjectInspector keyOI = ((MapObjectInspector)inspector).getMapKeyObjectInspector(); + ObjectInspector valueOI = ((MapObjectInspector)inspector).getMapValueObjectInspector(); + if (keyOI instanceof PrimitiveObjectInspector && valueOI instanceof PrimitiveObjectInspector) { + return toMapConstDesc(colInfo, inspector, keyOI, valueOI); + } + } + if (inspector instanceof ConstantObjectInspector && inspector instanceof StructObjectInspector) { + boolean allPrimitive = true; + List fields = ((StructObjectInspector)inspector).getAllStructFieldRefs(); + for (StructField field : fields) { + allPrimitive &= field.getFieldObjectInspector() instanceof PrimitiveObjectInspector; + } + if (allPrimitive) { + return toStructConstDesc(colInfo, inspector, fields); + } + } + // non-constant or non-primitive constants + ExprNodeColumnDesc column = new ExprNodeColumnDesc(colInfo); + column.setSkewedCol(colInfo.isSkewedCol()); + return column; + } + + private static ExprNodeConstantDesc toPrimitiveConstDesc(ColumnInfo colInfo, ObjectInspector inspector) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector; + Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue(); + ExprNodeConstantDesc constantExpr = + new ExprNodeConstantDesc(colInfo.getType(), poi.getPrimitiveJavaObject(constant)); + constantExpr.setFoldedFromCol(colInfo.getInternalName()); + constantExpr.setFoldedFromTab(colInfo.getTabAlias()); + return constantExpr; + } + + private static ExprNodeConstantDesc toListConstDesc(ColumnInfo colInfo, ObjectInspector inspector, + ObjectInspector listElementOI) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector)listElementOI; + List values = (List)((ConstantObjectInspector) inspector).getWritableConstantValue(); + List constant = new ArrayList(); + for (Object o : values) { + constant.add(poi.getPrimitiveJavaObject(o)); + } + + ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); + constantExpr.setFoldedFromCol(colInfo.getInternalName()); + constantExpr.setFoldedFromTab(colInfo.getTabAlias()); + return constantExpr; + } + + private static ExprNodeConstantDesc toMapConstDesc(ColumnInfo colInfo, ObjectInspector inspector, + ObjectInspector keyOI, ObjectInspector valueOI) { + PrimitiveObjectInspector keyPoi = (PrimitiveObjectInspector)keyOI; + PrimitiveObjectInspector valuePoi = (PrimitiveObjectInspector)valueOI; + Map values = (Map)((ConstantObjectInspector) inspector).getWritableConstantValue(); + Map constant = new LinkedHashMap(); + for (Map.Entry e : values.entrySet()) { + constant.put(keyPoi.getPrimitiveJavaObject(e.getKey()), valuePoi.getPrimitiveJavaObject(e.getValue())); + } + + ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); + constantExpr.setFoldedFromCol(colInfo.getInternalName()); + constantExpr.setFoldedFromTab(colInfo.getTabAlias()); + return constantExpr; + } + + private static ExprNodeConstantDesc toStructConstDesc(ColumnInfo colInfo, ObjectInspector inspector, + List fields) { + List values = (List)((ConstantObjectInspector) inspector).getWritableConstantValue(); + List constant = new ArrayList(); + for (int i = 0; i < values.size(); i++) { + Object value = values.get(i); + PrimitiveObjectInspector fieldPoi = (PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector(); + constant.add(fieldPoi.getPrimitiveJavaObject(value)); + } + ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); + constantExpr.setFoldedFromCol(colInfo.getInternalName()); + constantExpr.setFoldedFromTab(colInfo.getTabAlias()); + return constantExpr; + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeColumnDesc createColumnRefExpr(ColumnInfo colInfo) { + return new ExprNodeColumnDesc(colInfo); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createNullConstantExpr() { + return new ExprNodeConstantDesc(TypeInfoFactory. + getPrimitiveTypeInfoFromPrimitiveWritable(NullWritable.class), null); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createBooleanConstantExpr(String v) { + Boolean b = v != null ? Boolean.valueOf(v) : null; + return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, v); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createBigintConstantExpr(String v) { + Long l = Long.valueOf(v); + return new ExprNodeConstantDesc(l); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntConstantExpr(String v) { + Integer i = Integer.valueOf(v); + return new ExprNodeConstantDesc(i); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createSmallintConstantExpr(String v) { + Short s = Short.valueOf(v); + return new ExprNodeConstantDesc(s); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createTinyintConstantExpr(String v) { + Byte b = Byte.valueOf(v); + return new ExprNodeConstantDesc(b); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createFloatConstantExpr(String v) { + Float f = Float.valueOf(v); + return new ExprNodeConstantDesc(f); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createDoubleConstantExpr(String v) { + Double d = Double.valueOf(v); + return new ExprNodeConstantDesc(d); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) { + HiveDecimal hd = HiveDecimal.create(value); + if (!allowNullValueConstantExpr && hd == null) { + return null; + } + return new ExprNodeConstantDesc(adjustType(hd), hd); + } + + @Override + protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) { + if (constantValue instanceof HiveDecimal) { + return adjustType((HiveDecimal) constantValue); + } + return targetType; + } + + private DecimalTypeInfo adjustType(HiveDecimal hd) { + // Note: the normalize() call with rounding in HiveDecimal will currently reduce the + // precision and scale of the value by throwing away trailing zeroes. This may or may + // not be desirable for the literals; however, this used to be the default behavior + // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. + int prec = 1; + int scale = 0; + if (hd != null) { + prec = hd.precision(); + scale = hd.scale(); + } + DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale); + return typeInfo; + } + + /** + * {@inheritDoc} + */ + @Override + protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue, + PrimitiveTypeInfo sourceType) { + if (constantValue instanceof Number || constantValue instanceof String) { + try { + PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { + return toBigDecimal(constantValue.toString()).intValueExact(); + } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { + return toBigDecimal(constantValue.toString()).longValueExact(); + } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return Double.valueOf(constantValue.toString()); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return Float.valueOf(constantValue.toString()); + } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { + return toBigDecimal(constantValue.toString()).byteValueExact(); + } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { + return toBigDecimal(constantValue.toString()).shortValueExact(); + } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { + return HiveDecimal.create(constantValue.toString()); + } + } catch (NumberFormatException | ArithmeticException nfe) { + LOG.trace("Failed to narrow type of constant", nfe); + return null; + } + } + + // Comparision of decimal and float/double happens in float/double. + if (constantValue instanceof HiveDecimal) { + HiveDecimal hiveDecimal = (HiveDecimal) constantValue; + + PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.doubleValue(); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.floatValue(); + } + return hiveDecimal; + } + + String constTypeInfoName = sourceType.getTypeName(); + if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) { + // because a comparison against a "string" will happen in "string" type. + // to avoid unintnetional comparisions in "string" + // constants which are representing char/varchar values must be converted to the + // appropriate type. + if (targetType instanceof CharTypeInfo) { + final String constValue = constantValue.toString(); + final int length = TypeInfoUtils.getCharacterLengthForType(targetType); + HiveChar newValue = new HiveChar(constValue, length); + HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH); + if (maxCharConst.equals(newValue)) { + return newValue; + } else { + return null; + } + } + if (targetType instanceof VarcharTypeInfo) { + final String constValue = constantValue.toString(); + final int length = TypeInfoUtils.getCharacterLengthForType(targetType); + HiveVarchar newValue = new HiveVarchar(constValue, length); + HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH); + if (maxCharConst.equals(newValue)) { + return newValue; + } else { + return null; + } + } + } + + return constantValue; + } + + private BigDecimal toBigDecimal(String val) { + if (!NumberUtils.isNumber(val)) { + throw new NumberFormatException("The given string is not a valid number: " + val); + } + return new BigDecimal(val.replaceAll("[dDfFlL]$", "")); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createStringConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, value); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createDateConstantExpr(String value) { + Date d = Date.valueOf(value); + return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, d); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createTimestampConstantExpr(String value) { + Timestamp t = Timestamp.valueOf(value); + return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, t); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) { + TimestampTZ t = TimestampTZUtil.parse(value); + return new ExprNodeConstantDesc(TypeInfoFactory.getTimestampTZTypeInfo(zoneId), t); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalYearMonthConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, + HiveIntervalYearMonth.valueOf(value)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalDayTimeConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, + HiveIntervalDayTime.valueOf(value)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalYearConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, + new HiveIntervalYearMonth(Integer.parseInt(value), 0)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalMonthConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, + new HiveIntervalYearMonth(0, Integer.parseInt(value))); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalDayConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, + new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalHourConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, + new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalMinuteConstantExpr(String value) { + return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, + new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0)); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createIntervalSecondConstantExpr(String value) { + BigDecimal bd = new BigDecimal(value); + BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger()); + BigDecimal bdNanos = bd.subtract(bdSeconds); + return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, + new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(), + bdNanos.multiply(NANOS_PER_SEC_BD).intValue())); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeConstantDesc createConstantExpr(TypeInfo typeInfo, Object constantValue) { + return new ExprNodeConstantDesc(typeInfo, constantValue); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeFieldDesc createNestedColumnRefExpr( + TypeInfo typeInfo, ExprNodeDesc expr, String fieldName, Boolean isList) { + return new ExprNodeFieldDesc(typeInfo, expr, fieldName, isList); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeGenericFuncDesc createFuncCallExpr(TypeInfo typeInfo, GenericUDF genericUDF, + List inputs) { + return new ExprNodeGenericFuncDesc(typeInfo, genericUDF, inputs); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeGenericFuncDesc createFuncCallExpr(GenericUDF genericUDF, + String funcText, List inputs) throws UDFArgumentException { + return ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, inputs); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeColumnListDesc createExprsListExpr() { + return new ExprNodeColumnListDesc(); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeColumnListDesc addExprToExprsList(ExprNodeDesc columnList, ExprNodeDesc expr) { + ExprNodeColumnListDesc l = (ExprNodeColumnListDesc) columnList; + l.addColumn(expr); + return l; + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isConstantExpr(ExprNodeDesc expr) { + return expr instanceof ExprNodeConstantDesc; + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isFuncCallExpr(ExprNodeDesc expr) { + return expr instanceof ExprNodeGenericFuncDesc; + } + + /** + * {@inheritDoc} + */ + @Override + protected Object getConstantValue(ExprNodeDesc expr) { + return ((ExprNodeConstantDesc) expr).getValue(); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isColumnRefExpr(ExprNodeDesc expr) { + return expr instanceof ExprNodeColumnDesc; + } + + /** + * {@inheritDoc} + */ + @Override + protected String getColumnName(ExprNodeDesc expr) { + return ((ExprNodeColumnDesc) expr).getColumn(); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isExprsListExpr(ExprNodeDesc expr) { + return expr instanceof ExprNodeColumnListDesc; + } + + /** + * {@inheritDoc} + */ + @Override + protected List getExprChildren(ExprNodeDesc expr) { + return expr.getChildren(); + } + + /** + * {@inheritDoc} + */ + @Override + protected TypeInfo getTypeInfo(ExprNodeDesc expr) { + return expr.getTypeInfo(); + } + + /** + * {@inheritDoc} + */ + @Override + protected List rewriteINIntoORFuncCallExpr(List inOperands) throws SemanticException { + return TypeCheckProcFactoryUtils.rewriteInToOR(inOperands); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isORFuncCallExpr(ExprNodeDesc expr) { + return FunctionRegistry.isOpOr(expr); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isANDFuncCallExpr(ExprNodeDesc expr) { + return FunctionRegistry.isOpAnd(expr); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isPOSITIVEFuncCallExpr(ExprNodeDesc expr) { + return FunctionRegistry.isOpPositive(expr); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeDesc setTypeInfo(ExprNodeDesc expr, TypeInfo type) { + expr.setTypeInfo(type); + return expr; + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List inputs) { + if (genericUDF instanceof GenericUDFWhen && inputs.size() == 3 && + inputs.get(1) instanceof ExprNodeConstantDesc && + inputs.get(2) instanceof ExprNodeConstantDesc) { + ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) inputs.get(1); + ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) inputs.get(2); + Object thenVal = constThen.getValue(); + Object elseVal = constElse.getValue(); + if (thenVal instanceof Boolean && elseVal instanceof Boolean) { + return true; + } + } + return false; + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeDesc foldExpr(ExprNodeDesc expr) { + if (expr instanceof ExprNodeGenericFuncDesc) { + return ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc) expr); + } + return expr; + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isSTRUCTFuncCallExpr(ExprNodeDesc expr) { + return ExprNodeDescUtils.isStructUDF(expr); + } + + /** + * {@inheritDoc} + */ + @Override + protected boolean isConstantStruct(ExprNodeDesc expr) { + return ExprNodeDescUtils.isConstantStruct(expr); + } + + /** + * {@inheritDoc} + */ + @Override + protected ExprNodeDesc createSubqueryExpr(TypeCheckCtx ctx, ASTNode expr, SubqueryType subqueryType, + ExprNodeDesc[] inputs) throws CalciteSubquerySemanticException { + // subqueryToRelNode might be null if subquery expression anywhere other than + // as expected in filter (where/having). We should throw an appropriate error + // message + Map subqueryToRelNode = ctx.getSubqueryToRelNode(); + if (subqueryToRelNode == null) { + throw new CalciteSubquerySemanticException(ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg( + " Currently SubQuery expressions are only allowed as " + + "Where and Having Clause predicates")); + } + + ASTNode subqueryOp = (ASTNode) expr.getChild(0); + RelNode subqueryRel = subqueryToRelNode.get(expr); + // For now because subquery is only supported in filter + // we will create subquery expression of boolean type + switch (subqueryType) { + case EXISTS: { + return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, + SubqueryType.EXISTS); + } + case IN: { + assert (inputs[2] != null); + ExprNodeDesc lhs = inputs[2]; + return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, + SubqueryType.IN, lhs); + } + case SCALAR: { + // only single subquery expr is supported + if (subqueryRel.getRowType().getFieldCount() != 1) { + throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg( + "More than one column expression in subquery")); + } + // figure out subquery expression column's type + TypeInfo subExprType = TypeConverter.convert(subqueryRel.getRowType().getFieldList().get(0).getType()); + return new ExprNodeSubQueryDesc(subExprType, subqueryRel, + SubqueryType.SCALAR); + } + case SOME: { + assert (inputs[2] != null); + ExprNodeDesc lhs = inputs[2]; + return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, + SubqueryType.SOME, lhs, (ASTNode) subqueryOp.getChild(1)); + } + case ALL: { + assert (inputs[2] != null); + ExprNodeDesc lhs = inputs[2]; + return new ExprNodeSubQueryDesc(TypeInfoFactory.booleanTypeInfo, subqueryRel, + SubqueryType.ALL, lhs, (ASTNode) subqueryOp.getChild(1)); + } + default: + return null; + } + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java new file mode 100644 index 0000000000..623b280437 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse.type; + +import java.util.Map; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; + +/** + * Class with utility methods to use typecheck processor factory + * functionality. + */ +public class ExprNodeTypeCheck { + + private ExprNodeTypeCheck() { + // Defeat instantiation + } + + /** + * Given an AST expression and a context, it will produce a map from AST nodes + * to Hive ExprNode. + */ + public static Map genExprNode(ASTNode expr, TypeCheckCtx tcCtx) + throws SemanticException { + TypeCheckProcFactory factory = + new TypeCheckProcFactory<>(new ExprNodeDescExprFactory()); + return factory.genExprNode(expr, tcCtx); + } + + /** + * Given an AST join expression and a context, it will produce a map from AST nodes + * to Hive ExprNode. + */ + public static Map genExprNodeJoinCond(ASTNode expr, TypeCheckCtx tcCtx) + throws SemanticException { + JoinCondTypeCheckProcFactory typeCheckProcFactory = + new JoinCondTypeCheckProcFactory<>(new ExprNodeDescExprFactory()); + return typeCheckProcFactory.genExprNode(expr, tcCtx); + } + + /** + * Returns the default processor to generate Hive ExprNode from AST nodes. + */ + public static TypeCheckProcFactory.DefaultExprProcessor getExprNodeDefaultExprProcessor() { + TypeCheckProcFactory factory = + new TypeCheckProcFactory<>(new ExprNodeDescExprFactory()); + return factory.getDefaultExprProcessor(); + } + + /** + * Transforms column information into the corresponding Hive ExprNode. + */ + public static ExprNodeDesc toExprNodeDesc(ColumnInfo columnInfo) { + ExprNodeDescExprFactory factory = new ExprNodeDescExprFactory(); + return factory.toExpr(columnInfo); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinCondTypeCheckProcFactory.java similarity index 78% rename from ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java rename to ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinCondTypeCheckProcFactory.java index 833989722e..d5652f7d4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinCondTypeCheckProcFactory.java @@ -15,11 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hive.ql.optimizer.calcite.translator; +package org.apache.hadoop.hive.ql.parse.type; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Stack; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -31,11 +30,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; + /** * JoinCondTypeCheckProcFactory is used by Calcite planner(CBO) to generate Join Conditions from Join Condition AST. @@ -51,17 +46,17 @@ * 2. Use Column Processing from TypeCheckProcFactory
* 3. Why not use GB expr ? */ -public class JoinCondTypeCheckProcFactory extends TypeCheckProcFactory { +public class JoinCondTypeCheckProcFactory extends TypeCheckProcFactory { - public static Map genExprNode(ASTNode expr, TypeCheckCtx tcCtx) - throws SemanticException { - return TypeCheckProcFactory.genExprNode(expr, tcCtx, new JoinCondTypeCheckProcFactory()); + protected JoinCondTypeCheckProcFactory(ExprFactory factory) { + // prevent instantiation + super(factory); } /** * Processor for table columns. */ - public static class JoinCondColumnExprProcessor extends ColumnExprProcessor { + public class JoinCondColumnExprProcessor extends ColumnExprProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, @@ -89,7 +84,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (!qualifiedAccess) { colInfo = getColInfo(ctx, null, tableOrCol, expr); // It's a column. - return new ExprNodeColumnDesc(colInfo); + return exprFactory.createColumnRefExpr(colInfo); } else if (hasTableAlias(ctx, tableOrCol, expr)) { return null; } else { @@ -98,7 +93,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } - private static boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNode expr) + private boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNode expr) throws SemanticException { int tblAliasCnt = 0; for (RowResolver rr : ctx.getInputRRList()) { @@ -113,7 +108,7 @@ private static boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNo return (tblAliasCnt == 1) ? true : false; } - private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, + private ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { ColumnInfo tmp; ColumnInfo cInfoToRet = null; @@ -138,14 +133,14 @@ private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, Strin * @return ColumnExprProcessor. */ @Override - public ColumnExprProcessor getColumnExprProcessor() { + protected ColumnExprProcessor getColumnExprProcessor() { return new JoinCondColumnExprProcessor(); } /** * The default processor for typechecking. */ - public static class JoinCondDefaultExprProcessor extends DefaultExprProcessor { + protected class JoinCondDefaultExprProcessor extends DefaultExprProcessor { @Override protected List getReferenceableColumnAliases(TypeCheckCtx ctx) { JoinTypeCheckCtx jCtx = (JoinTypeCheckCtx) ctx; @@ -158,24 +153,25 @@ public ColumnExprProcessor getColumnExprProcessor() { } @Override - protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr, + protected T processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr, Object... nodeOutputs) throws SemanticException { String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0) .getText()); // NOTE: tableAlias must be a valid non-ambiguous table alias, // because we've checked that in TOK_TABLE_OR_COL's process method. ColumnInfo colInfo = getColInfo((JoinTypeCheckCtx) ctx, tableAlias, - ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString(), expr); + exprFactory.getConstantValue((T) nodeOutputs[1]).toString(), expr); if (colInfo == null) { ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr); return null; } - return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), tableAlias, - colInfo.getIsVirtualCol()); + ColumnInfo newColumnInfo = new ColumnInfo(colInfo); + newColumnInfo.setTabAlias(tableAlias); + return exprFactory.createColumnRefExpr(newColumnInfo); } - private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, + private ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias, ASTNode expr) throws SemanticException { ColumnInfo tmp; ColumnInfo cInfoToRet = null; @@ -200,7 +196,7 @@ private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, Strin * @return DefaultExprProcessor. */ @Override - public DefaultExprProcessor getDefaultExprProcessor() { + protected DefaultExprProcessor getDefaultExprProcessor() { return new JoinCondDefaultExprProcessor(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinTypeCheckCtx.java similarity index 95% rename from ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java rename to ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinTypeCheckCtx.java index 871518c713..8832c772eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/JoinTypeCheckCtx.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hive.ql.optimizer.calcite.translator; +package org.apache.hadoop.hive.ql.parse.type; import java.util.List; import org.apache.hadoop.hive.ql.parse.JoinType; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; import com.google.common.collect.ImmutableList; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckCtx.java similarity index 97% rename from ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java rename to ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckCtx.java index 12e7ae62f4..67875cc31f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckCtx.java @@ -16,10 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.parse.type; import org.apache.calcite.rel.RelNode; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.UnparseTranslator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java new file mode 100644 index 0000000000..571717e0ea --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java @@ -0,0 +1,1560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse.type; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.FunctionInfo; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.lib.CostLessRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.SubqueryExpressionWalker; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.SubqueryType; +import org.apache.hadoop.hive.ql.udf.SettableUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.SetMultimap; + +/** + * The Factory for creating typecheck processors. The typecheck processors are + * used to processes the syntax trees for expressions and convert them into + * expression Node Descriptor trees. They also introduce the correct conversion + * functions to do proper implicit conversion. + * + * At instantiation time, a expression factory needs to be provided to this class. + */ +public class TypeCheckProcFactory { + + static final Logger LOG = LoggerFactory.getLogger( + TypeCheckProcFactory.class.getName()); + + static final HashMap SPECIAL_UNARY_OPERATOR_TEXT_MAP; + static final HashMap CONVERSION_FUNCTION_TEXT_MAP; + static final HashSet WINDOWING_TOKENS; + + static { + SPECIAL_UNARY_OPERATOR_TEXT_MAP = new HashMap<>(); + SPECIAL_UNARY_OPERATOR_TEXT_MAP.put(HiveParser.PLUS, "positive"); + SPECIAL_UNARY_OPERATOR_TEXT_MAP.put(HiveParser.MINUS, "negative"); + + CONVERSION_FUNCTION_TEXT_MAP = new HashMap(); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_BOOLEAN, + serdeConstants.BOOLEAN_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_TINYINT, + serdeConstants.TINYINT_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_SMALLINT, + serdeConstants.SMALLINT_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_INT, + serdeConstants.INT_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_BIGINT, + serdeConstants.BIGINT_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_FLOAT, + serdeConstants.FLOAT_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_DOUBLE, + serdeConstants.DOUBLE_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_STRING, + serdeConstants.STRING_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_CHAR, + serdeConstants.CHAR_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_VARCHAR, + serdeConstants.VARCHAR_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_BINARY, + serdeConstants.BINARY_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_DATE, + serdeConstants.DATE_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_TIMESTAMP, + serdeConstants.TIMESTAMP_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_TIMESTAMPLOCALTZ, + serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_INTERVAL_YEAR_MONTH, + serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_INTERVAL_DAY_TIME, + serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_DECIMAL, + serdeConstants.DECIMAL_TYPE_NAME); + + WINDOWING_TOKENS = new HashSet(); + WINDOWING_TOKENS.add(HiveParser.KW_OVER); + WINDOWING_TOKENS.add(HiveParser.TOK_PARTITIONINGSPEC); + WINDOWING_TOKENS.add(HiveParser.TOK_DISTRIBUTEBY); + WINDOWING_TOKENS.add(HiveParser.TOK_SORTBY); + WINDOWING_TOKENS.add(HiveParser.TOK_CLUSTERBY); + WINDOWING_TOKENS.add(HiveParser.TOK_WINDOWSPEC); + WINDOWING_TOKENS.add(HiveParser.TOK_WINDOWRANGE); + WINDOWING_TOKENS.add(HiveParser.TOK_WINDOWVALUES); + WINDOWING_TOKENS.add(HiveParser.KW_UNBOUNDED); + WINDOWING_TOKENS.add(HiveParser.KW_PRECEDING); + WINDOWING_TOKENS.add(HiveParser.KW_FOLLOWING); + WINDOWING_TOKENS.add(HiveParser.KW_CURRENT); + WINDOWING_TOKENS.add(HiveParser.TOK_TABSORTCOLNAMEASC); + WINDOWING_TOKENS.add(HiveParser.TOK_TABSORTCOLNAMEDESC); + WINDOWING_TOKENS.add(HiveParser.TOK_NULLS_FIRST); + WINDOWING_TOKENS.add(HiveParser.TOK_NULLS_LAST); + } + + + /** + * Factory that will be used to create the different expressions. + */ + protected final ExprFactory exprFactory; + + protected TypeCheckProcFactory(ExprFactory exprFactory) { + this.exprFactory = exprFactory; + } + + protected Map genExprNode(ASTNode expr, TypeCheckCtx tcCtx) throws SemanticException { + // Create the walker, the rules dispatcher and the context. + // create a walker which walks the tree in a DFS manner while maintaining + // the operator stack. The dispatcher + // generates the plan from the operator tree + + SetMultimap astNodeToProcessor = HashMultimap.create(); + astNodeToProcessor.put(HiveParser.TOK_NULL, getNullExprProcessor()); + + astNodeToProcessor.put(HiveParser.Number, getNumExprProcessor()); + astNodeToProcessor.put(HiveParser.IntegralLiteral, getNumExprProcessor()); + astNodeToProcessor.put(HiveParser.NumberLiteral, getNumExprProcessor()); + + astNodeToProcessor.put(HiveParser.Identifier, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.StringLiteral, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_CHARSETLITERAL, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_STRINGLITERALSEQUENCE, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_IF, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_CASE, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_WHEN, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_IN, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_ARRAY, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_MAP, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_STRUCT, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_EXISTS, getStrExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_SUBQUERY_OP_NOTIN, getStrExprProcessor()); + + astNodeToProcessor.put(HiveParser.KW_TRUE, getBoolExprProcessor()); + astNodeToProcessor.put(HiveParser.KW_FALSE, getBoolExprProcessor()); + + astNodeToProcessor.put(HiveParser.TOK_DATELITERAL, getDateTimeExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_TIMESTAMPLITERAL, getDateTimeExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_TIMESTAMPLOCALTZLITERAL, getDateTimeExprProcessor()); + + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_YEAR_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_MONTH_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_DAY_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_HOUR_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_MINUTE_LITERAL, getIntervalExprProcessor()); + astNodeToProcessor.put(HiveParser.TOK_INTERVAL_SECOND_LITERAL, getIntervalExprProcessor()); + + astNodeToProcessor.put(HiveParser.TOK_TABLE_OR_COL, getColumnExprProcessor()); + + astNodeToProcessor.put(HiveParser.TOK_SUBQUERY_EXPR, getSubQueryExprProcessor()); + + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new CostLessRuleDispatcher(getDefaultExprProcessor(), + astNodeToProcessor, tcCtx); + GraphWalker ogw = new SubqueryExpressionWalker(disp); + + // Create a list of top nodes + ArrayList topNodes = Lists.newArrayList(expr); + HashMap nodeOutputs = new LinkedHashMap(); + ogw.startWalking(topNodes, nodeOutputs); + + return convert(nodeOutputs); + } + + // temporary type-safe casting + protected Map convert(Map outputs) { + Map converted = new LinkedHashMap<>(); + for (Map.Entry entry : outputs.entrySet()) { + if (entry.getKey() instanceof ASTNode && + (entry.getValue() == null || exprFactory.isExprInstance(entry.getValue()))) { + converted.put((ASTNode)entry.getKey(), (T) entry.getValue()); + } else { + LOG.warn("Invalid type entry " + entry); + } + } + return converted; + } + + /** + * Processor for processing NULL expression. + */ + public class NullExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + return exprFactory.createNullConstantExpr(); + } + + } + + /** + * Factory method to get NullExprProcessor. + * + * @return NullExprProcessor. + */ + protected NullExprProcessor getNullExprProcessor() { + return new NullExprProcessor(); + } + + /** + * Processor for processing numeric constants. + */ + public class NumExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + T result = null; + ASTNode expr = (ASTNode) nd; + try { + if (expr.getText().endsWith("L")) { + // Literal bigint. + result = exprFactory.createBigintConstantExpr( + expr.getText().substring(0, expr.getText().length() - 1)); + } else if (expr.getText().endsWith("S")) { + // Literal smallint. + result = exprFactory.createSmallintConstantExpr( + expr.getText().substring(0, expr.getText().length() - 1)); + } else if (expr.getText().endsWith("Y")) { + // Literal tinyint. + result = exprFactory.createTinyintConstantExpr( + expr.getText().substring(0, expr.getText().length() - 1)); + } else if (expr.getText().endsWith("F")) { + // Literal float. + result = exprFactory.createFloatConstantExpr( + expr.getText().substring(0, expr.getText().length() - 1)); + } else if (expr.getText().endsWith("D")) { + // Literal double. + result = exprFactory.createDoubleConstantExpr( + expr.getText().substring(0, expr.getText().length() - 1)); + } else if (expr.getText().endsWith("BD")) { + // Literal decimal + result = exprFactory.createDecimalConstantExpr( + expr.getText().substring(0, expr.getText().length() - 2), true); + } else { + // Default behavior + result = exprFactory.createConstantExpr(expr.getText()); + } + } catch (NumberFormatException e) { + // do nothing here, we will throw an exception in the following block + } + if (result == null) { + throw new SemanticException(ErrorMsg.INVALID_NUMERICAL_CONSTANT.getMsg(expr)); + } + return result; + } + + } + + /** + * Factory method to get NumExprProcessor. + * + * @return NumExprProcessor. + */ + protected NumExprProcessor getNumExprProcessor() { + return new NumExprProcessor(); + } + + /** + * Processor for processing string constants. + */ + public class StrExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + String str = null; + + switch (expr.getToken().getType()) { + case HiveParser.StringLiteral: + str = BaseSemanticAnalyzer.unescapeSQLString(expr.getText()); + break; + case HiveParser.TOK_STRINGLITERALSEQUENCE: + StringBuilder sb = new StringBuilder(); + for (Node n : expr.getChildren()) { + sb.append( + BaseSemanticAnalyzer.unescapeSQLString(((ASTNode) n).getText())); + } + str = sb.toString(); + break; + case HiveParser.TOK_CHARSETLITERAL: + str = BaseSemanticAnalyzer.charSetString(expr.getChild(0).getText(), + expr.getChild(1).getText()); + break; + default: + // HiveParser.identifier | HiveParse.KW_IF | HiveParse.KW_LEFT | + // HiveParse.KW_RIGHT + str = BaseSemanticAnalyzer.unescapeIdentifier(expr.getText().toLowerCase()); + break; + } + return exprFactory.createStringConstantExpr(str); + } + + } + + /** + * Factory method to get StrExprProcessor. + * + * @return StrExprProcessor. + */ + protected StrExprProcessor getStrExprProcessor() { + return new StrExprProcessor(); + } + + /** + * Processor for boolean constants. + */ + public class BoolExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + String bool = null; + + switch (expr.getToken().getType()) { + case HiveParser.KW_TRUE: + bool = Boolean.TRUE.toString(); + break; + case HiveParser.KW_FALSE: + bool = Boolean.FALSE.toString(); + break; + default: + assert false; + } + return exprFactory.createBooleanConstantExpr(bool); + } + + } + + /** + * Factory method to get BoolExprProcessor. + * + * @return BoolExprProcessor. + */ + protected BoolExprProcessor getBoolExprProcessor() { + return new BoolExprProcessor(); + } + + /** + * Processor for date constants. + */ + public class DateTimeExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + String timeString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); + + // Get the string value and convert to a Date value. + try { + if (expr.getType() == HiveParser.TOK_DATELITERAL) { + return exprFactory.createDateConstantExpr(timeString); + } + if (expr.getType() == HiveParser.TOK_TIMESTAMPLITERAL) { + return exprFactory.createTimestampConstantExpr(timeString); + } + if (expr.getType() == HiveParser.TOK_TIMESTAMPLOCALTZLITERAL) { + HiveConf conf; + try { + conf = Hive.get().getConf(); + } catch (HiveException e) { + throw new SemanticException(e); + } + return exprFactory.createTimestampLocalTimeZoneConstantExpr(timeString, conf.getLocalTimeZone()); + } + throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); + } catch (Exception err) { + throw new SemanticException( + "Unable to convert time literal '" + timeString + "' to time value.", err); + } + } + } + + /** + * Factory method to get DateExprProcessor. + * + * @return DateExprProcessor. + */ + protected DateTimeExprProcessor getDateTimeExprProcessor() { + return new DateTimeExprProcessor(); + } + + /** + * Processor for interval constants. + */ + public class IntervalExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + String intervalString = BaseSemanticAnalyzer.stripQuotes(expr.getText()); + + // Get the string value and convert to a Interval value. + try { + switch (expr.getType()) { + case HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL: + return exprFactory.createIntervalYearMonthConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL: + return exprFactory.createIntervalDayTimeConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_YEAR_LITERAL: + return exprFactory.createIntervalYearConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_MONTH_LITERAL: + return exprFactory.createIntervalMonthConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_DAY_LITERAL: + return exprFactory.createIntervalDayConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_HOUR_LITERAL: + return exprFactory.createIntervalHourConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_MINUTE_LITERAL: + return exprFactory.createIntervalMinuteConstantExpr(intervalString); + case HiveParser.TOK_INTERVAL_SECOND_LITERAL: + return exprFactory.createIntervalSecondConstantExpr(intervalString); + default: + throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); + } + } catch (Exception err) { + throw new SemanticException( + "Unable to convert interval literal '" + intervalString + "' to interval value.", err); + } + } + } + + /** + * Factory method to get IntervalExprProcessor. + * + * @return IntervalExprProcessor. + */ + protected IntervalExprProcessor getIntervalExprProcessor() { + return new IntervalExprProcessor(); + } + + /** + * Processor for table columns. + */ + public class ColumnExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + ASTNode expr = (ASTNode) nd; + ASTNode parent = stack.size() > 1 ? (ASTNode) stack.get(stack.size() - 2) : null; + RowResolver input = ctx.getInputRR(); + if (input == null) { + ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr); + return null; + } + + if (expr.getType() != HiveParser.TOK_TABLE_OR_COL) { + ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr); + return null; + } + + assert (expr.getChildCount() == 1); + String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr + .getChild(0).getText()); + + boolean isTableAlias = input.hasTableAlias(tableOrCol); + ColumnInfo colInfo = null; + try { + colInfo = input.get(null, tableOrCol); + } catch (SemanticException semanticException) { + if (!isTableAlias || parent == null || parent.getType() != HiveParser.DOT) { + throw semanticException; + } + } + // try outer row resolver + if (ctx.getOuterRR() != null && colInfo == null && !isTableAlias) { + RowResolver outerRR = ctx.getOuterRR(); + isTableAlias = outerRR.hasTableAlias(tableOrCol); + colInfo = outerRR.get(null, tableOrCol); + } + + if (isTableAlias) { + if (colInfo != null) { + if (parent != null && parent.getType() == HiveParser.DOT) { + // It's a table alias. + return null; + } + // It's a column. + return exprFactory.toExpr(colInfo); + } else { + // It's a table alias. + // We will process that later in DOT. + return null; + } + } else { + if (colInfo == null) { + // It's not a column or a table alias. + if (input.getIsExprResolver()) { + ASTNode exprNode = expr; + if (!stack.empty()) { + ASTNode tmp = (ASTNode) stack.pop(); + if (!stack.empty()) { + exprNode = (ASTNode) stack.peek(); + } + stack.push(tmp); + } + ctx.setError(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(exprNode), expr); + return null; + } else { + List possibleColumnNames = input.getReferenceableColumnAliases(tableOrCol, -1); + String reason = String.format("(possible column names are: %s)", + StringUtils.join(possibleColumnNames, ", ")); + ctx.setError(ErrorMsg.INVALID_TABLE_OR_COLUMN.getMsg(expr.getChild(0), reason), + expr); + LOG.debug(ErrorMsg.INVALID_TABLE_OR_COLUMN.toString() + ":" + + input.toString()); + return null; + } + } else { + // It's a column. + return exprFactory.toExpr(colInfo); + } + } + } + + } + + /** + * Factory method to get ColumnExprProcessor. + * + * @return ColumnExprProcessor. + */ + protected ColumnExprProcessor getColumnExprProcessor() { + return new ColumnExprProcessor(); + } + + /** + * The default processor for typechecking. + */ + public class DefaultExprProcessor implements NodeProcessor { + + protected boolean isRedundantConversionFunction(ASTNode expr, + boolean isFunction, List children) { + if (!isFunction) { + return false; + } + // conversion functions take a single parameter + if (children.size() != 1) { + return false; + } + String funcText = CONVERSION_FUNCTION_TEXT_MAP.get(((ASTNode) expr + .getChild(0)).getType()); + // not a conversion function + if (funcText == null) { + return false; + } + // return true when the child type and the conversion target type is the + // same + return exprFactory.getTypeInfo(children.get(0)).getTypeName().equalsIgnoreCase(funcText); + } + + /** + * This function create an ExprNodeDesc for a UDF function given the + * children (arguments). It will insert implicit type conversion functions + * if necessary. + * + * @throws UDFArgumentException + */ + public T getFuncExprNodeDescWithUdfData(String udfName, TypeInfo typeInfo, + T... children) throws UDFArgumentException { + + FunctionInfo fi; + try { + fi = FunctionRegistry.getFunctionInfo(udfName); + } catch (SemanticException e) { + throw new UDFArgumentException(e); + } + if (fi == null) { + throw new UDFArgumentException(udfName + " not found."); + } + + GenericUDF genericUDF = fi.getGenericUDF(); + if (genericUDF == null) { + throw new UDFArgumentException(udfName + + " is an aggregation function or a table function."); + } + + // Add udfData to UDF if necessary + if (typeInfo != null) { + if (genericUDF instanceof SettableUDF) { + ((SettableUDF) genericUDF).setTypeInfo(typeInfo); + } + } + + List childrenList = new ArrayList<>(children.length); + + childrenList.addAll(Arrays.asList(children)); + return exprFactory.createFuncCallExpr(genericUDF, null, childrenList); + } + + public T getFuncExprNodeDesc(String udfName, T... children) throws UDFArgumentException { + return getFuncExprNodeDescWithUdfData(udfName, null, children); + } + + /** + * @param column column expression to convert + * @param tableFieldTypeInfo TypeInfo to convert to + * @return Expression converting column to the type specified by tableFieldTypeInfo + */ + public T createConversionCast(T column, PrimitiveTypeInfo tableFieldTypeInfo) + throws SemanticException { + // Get base type, since type string may be parameterized + String baseType = TypeInfoUtils.getBaseName(tableFieldTypeInfo.getTypeName()); + + // If the type cast UDF is for a parameterized type, then it should implement + // the SettableUDF interface so that we can pass in the params. + // Not sure if this is the cleanest solution, but there does need to be a way + // to provide the type params to the type cast. + return getDefaultExprProcessor().getFuncExprNodeDescWithUdfData(baseType, tableFieldTypeInfo, column); + } + + protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, FunctionInfo fi, + List children, GenericUDF genericUDF) throws SemanticException { + // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't + // supported + if (fi.getGenericUDTF() != null) { + throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg()); + } + // UDAF in filter condition, group-by caluse, param of funtion, etc. + if (fi.getGenericUDAFResolver() != null) { + if (isFunction) { + throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg((ASTNode) expr + .getChild(0))); + } else { + throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr)); + } + } + if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) { + if (FunctionRegistry.isStateful(genericUDF)) { + throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg()); + } + } + } + + protected void insertCast(String funcText, List children) throws SemanticException { + // substring, concat UDFs expect first argument as string. Therefore this method inserts explicit cast + // to cast the first operand to string + if (funcText.equals("substring") || funcText.equals("concat")) { + if (children.size() > 0 && !isStringType(exprFactory.getTypeInfo(children.get(0)))) { + T newColumn = createConversionCast(children.get(0), TypeInfoFactory.stringTypeInfo); + children.set(0, newColumn); + } + } + } + + protected T getXpathOrFuncExprNodeDesc(ASTNode expr, + boolean isFunction, List children, TypeCheckCtx ctx) + throws SemanticException, UDFArgumentException { + // return the child directly if the conversion is redundant. + if (isRedundantConversionFunction(expr, isFunction, children)) { + assert (children.size() == 1); + assert (children.get(0) != null); + return children.get(0); + } + String funcText = getFunctionText(expr, isFunction); + T desc; + if (funcText.equals(".")) { + // "." : FIELD Expression + + assert (children.size() == 2); + // Only allow constant field name for now + assert (exprFactory.isConstantExpr(children.get(1))); + T object = children.get(0); + + // Calculate result TypeInfo + String fieldNameString = exprFactory.getConstantValue(children.get(1)).toString(); + TypeInfo objectTypeInfo = exprFactory.getTypeInfo(object); + + // Allow accessing a field of list element structs directly from a list + boolean isList = (objectTypeInfo.getCategory() == ObjectInspector.Category.LIST); + if (isList) { + objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); + } + if (objectTypeInfo.getCategory() != Category.STRUCT) { + throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); + } + TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); + if (isList) { + t = TypeInfoFactory.getListTypeInfo(t); + } + + desc = exprFactory.createNestedColumnRefExpr(t, children.get(0), fieldNameString, isList); + } else if (funcText.equals("[")) { + // "[]" : LSQUARE/INDEX Expression + if (!ctx.getallowIndexExpr()) { + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr)); + } + + assert (children.size() == 2); + + // Check whether this is a list or a map + TypeInfo myt = exprFactory.getTypeInfo(children.get(0)); + + if (myt.getCategory() == Category.LIST) { + // Only allow integer index for now + if (!TypeInfoUtils.implicitConvertible(exprFactory.getTypeInfo(children.get(1)), + TypeInfoFactory.intTypeInfo)) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage( + expr, ErrorMsg.INVALID_ARRAYINDEX_TYPE.getMsg())); + } + + // Calculate TypeInfo + TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); + desc = exprFactory.createFuncCallExpr(t, FunctionRegistry.getGenericUDFForIndex(), children); + } else if (myt.getCategory() == Category.MAP) { + if (!TypeInfoUtils.implicitConvertible(exprFactory.getTypeInfo(children.get(1)), + ((MapTypeInfo) myt).getMapKeyTypeInfo())) { + throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE + .getMsg(expr)); + } + // Calculate TypeInfo + TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); + desc = exprFactory.createFuncCallExpr(t, FunctionRegistry.getGenericUDFForIndex(), children); + } else { + throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, myt.getTypeName())); + } + } else { + // other operators or functions + FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcText); + + if (fi == null) { + if (isFunction) { + throw new SemanticException(ErrorMsg.INVALID_FUNCTION + .getMsg((ASTNode) expr.getChild(0))); + } else { + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr)); + } + } + + // getGenericUDF() actually clones the UDF. Just call it once and reuse. + GenericUDF genericUDF = fi.getGenericUDF(); + + if (!fi.isNative()) { + ctx.getUnparseTranslator().addIdentifierTranslation( + (ASTNode) expr.getChild(0)); + } + + // Handle type casts that may contain type parameters + if (isFunction) { + ASTNode funcNameNode = (ASTNode) expr.getChild(0); + switch (funcNameNode.getType()) { + case HiveParser.TOK_CHAR: + // Add type params + CharTypeInfo charTypeInfo = ParseUtils.getCharTypeInfo(funcNameNode); + if (genericUDF != null) { + ((SettableUDF) genericUDF).setTypeInfo(charTypeInfo); + } + break; + case HiveParser.TOK_VARCHAR: + VarcharTypeInfo varcharTypeInfo = ParseUtils.getVarcharTypeInfo(funcNameNode); + if (genericUDF != null) { + ((SettableUDF) genericUDF).setTypeInfo(varcharTypeInfo); + } + break; + case HiveParser.TOK_TIMESTAMPLOCALTZ: + TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = new TimestampLocalTZTypeInfo(); + HiveConf conf; + try { + conf = Hive.get().getConf(); + } catch (HiveException e) { + throw new SemanticException(e); + } + timestampLocalTZTypeInfo.setTimeZone(conf.getLocalTimeZone()); + if (genericUDF != null) { + ((SettableUDF) genericUDF).setTypeInfo(timestampLocalTZTypeInfo); + } + break; + case HiveParser.TOK_DECIMAL: + DecimalTypeInfo decTypeInfo = ParseUtils.getDecimalTypeTypeInfo(funcNameNode); + if (genericUDF != null) { + ((SettableUDF) genericUDF).setTypeInfo(decTypeInfo); + } + break; + default: + // Do nothing + break; + } + } + + insertCast(funcText, children); + + validateUDF(expr, isFunction, ctx, fi, children, genericUDF); + + // Try to infer the type of the constant only if there are two + // nodes, one of them is column and the other is numeric const + if (genericUDF instanceof GenericUDFBaseCompare + && children.size() == 2 + && ((children.get(0) instanceof ExprNodeConstantDesc + && children.get(1) instanceof ExprNodeColumnDesc) + || (children.get(0) instanceof ExprNodeColumnDesc + && children.get(1) instanceof ExprNodeConstantDesc))) { + + int constIdx = children.get(0) instanceof ExprNodeConstantDesc ? 0 : 1; + + T constChild = children.get(constIdx); + T columnChild = children.get(1 - constIdx); + + final PrimitiveTypeInfo colTypeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(exprFactory.getTypeInfo(columnChild).getTypeName().toLowerCase()); + T newChild = interpretNodeAs(colTypeInfo, constChild); + if (newChild == null) { + // non-interpretable as target type... + // TODO: all comparisons with null should result in null + if (genericUDF instanceof GenericUDFOPEqual + && !(genericUDF instanceof GenericUDFOPEqualNS)) { + return exprFactory.createBooleanConstantExpr(null); + } + } else { + children.set(constIdx, newChild); + } + } + if (genericUDF instanceof GenericUDFIn) { + + T columnDesc = children.get(0); + List outputOpList = children.subList(1, children.size()); + List inOperands = new ArrayList<>(outputOpList); + outputOpList.clear(); + + boolean hasNullValue = false; + for (T oldChild : inOperands) { + if (oldChild == null) { + hasNullValue = true; + continue; + } + T newChild = interpretNodeAsStruct(columnDesc, oldChild); + if (newChild == null) { + hasNullValue = true; + continue; + } + outputOpList.add(newChild); + } + + if (hasNullValue) { + T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null); + if (outputOpList.size() == 0) { + // we have found only null values...remove the IN ; it will be null all the time. + return nullConst; + } + outputOpList.add(nullConst); + } + + if (!ctx.isCBOExecuted()) { + + HiveConf conf; + try { + conf = Hive.get().getConf(); + } catch (HiveException e) { + throw new SemanticException(e); + } + if (children.size() <= HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES)) { + List orOperands = exprFactory.rewriteINIntoORFuncCallExpr(children); + if (orOperands != null) { + if (orOperands.size() == 1) { + orOperands.add(exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString())); + } + funcText = "or"; + genericUDF = new GenericUDFOPOr(); + children.clear(); + children.addAll(orOperands); + } + } + } + } + if (genericUDF instanceof GenericUDFOPOr) { + // flatten OR + List childrenList = new ArrayList<>(children.size()); + for (T child : children) { + if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(exprFactory.getTypeInfo(child))) { + child = exprFactory.setTypeInfo(child, TypeInfoFactory.getPrimitiveTypeInfo("boolean")); + } + if (exprFactory.isORFuncCallExpr(child)) { + childrenList.addAll(exprFactory.getExprChildren(child)); + } else { + childrenList.add(child); + } + } + desc = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList); + } else if (genericUDF instanceof GenericUDFOPAnd) { + // flatten AND + List childrenList = new ArrayList<>(children.size()); + for (T child : children) { + if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(exprFactory.getTypeInfo(child))) { + child = exprFactory.setTypeInfo(child, TypeInfoFactory.getPrimitiveTypeInfo("boolean")); + } + if (exprFactory.isANDFuncCallExpr(child)) { + childrenList.addAll(exprFactory.getExprChildren(child)); + } else { + childrenList.add(child); + } + } + desc = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList); + } else if (ctx.isFoldExpr() && exprFactory.canConvertCASEIntoCOALESCEFuncCallExpr(genericUDF, children)) { + // Rewrite CASE into COALESCE + desc = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), null, + Lists.newArrayList(children.get(0), exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString()))); + if (Boolean.FALSE.equals(exprFactory.getConstantValue(children.get(1)))) { + desc = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), null, Lists.newArrayList(desc)); + } + } else { + desc = exprFactory.createFuncCallExpr(genericUDF, funcText, children); + } + + // If the function is deterministic and the children are constants, + // we try to fold the expression to remove e.g. cast on constant + if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(desc) && + FunctionRegistry.isConsistentWithinQuery(genericUDF) && + exprFactory.isAllConstants(children)) { + T constantExpr = exprFactory.foldExpr(desc); + if (constantExpr != null) { + desc = constantExpr; + } + } + } + // UDFOPPositive is a no-op. + // However, we still create it, and then remove it here, to make sure we + // only allow + // "+" for numeric types. + if (exprFactory.isPOSITIVEFuncCallExpr(desc)) { + assert (exprFactory.getExprChildren(desc).size() == 1); + desc = exprFactory.getExprChildren(desc).get(0); + } + assert (desc != null); + return desc; + } + + /** + * Interprets the given value as columnDesc if possible + */ + private T interpretNodeAsStruct(T columnDesc, T valueDesc) + throws SemanticException { + if (exprFactory.isColumnRefExpr(columnDesc)) { + final PrimitiveTypeInfo typeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(exprFactory.getTypeInfo(columnDesc).getTypeName().toLowerCase()); + return interpretNodeAs(typeInfo, valueDesc); + } + if (exprFactory.isSTRUCTFuncCallExpr(columnDesc) && exprFactory.isConstantStruct(valueDesc)) { + List columnChilds = exprFactory.getExprChildren(columnDesc); + ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; + StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); + ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); + ArrayList newStructFieldInfos = new ArrayList<>(); + + if (columnChilds.size() != structFieldInfos.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos)); + } + List oldValues = (List) valueConstDesc.getValue(); + List newValues = new ArrayList<>(); + for (int i = 0; i < columnChilds.size(); i++) { + newStructFieldInfos.add(exprFactory.getTypeInfo(columnChilds.get(i))); + Object newValue = exprFactory.interpretConstantAsPrimitive( + (PrimitiveTypeInfo) exprFactory.getTypeInfo(columnChilds.get(i)), + oldValues.get(i), + (PrimitiveTypeInfo) structFieldInfos.get(i)); + newValues.add(newValue); + } + StructTypeInfo sti = new StructTypeInfo(); + sti.setAllStructFieldTypeInfos(newStructFieldInfos); + sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); + return exprFactory.createConstantExpr(sti, newValues); + + } + if (exprFactory.isSTRUCTFuncCallExpr(columnDesc) && exprFactory.isSTRUCTFuncCallExpr(valueDesc)) { + List columnChilds = exprFactory.getExprChildren(columnDesc); + List valueChilds = exprFactory.getExprChildren(valueDesc); + if (columnChilds.size() != valueChilds.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds)); + } + List oldValueChilds = new ArrayList<>(valueChilds); + valueChilds.clear(); + for (int i = 0; i < oldValueChilds.size(); i++) { + T newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i)); + valueChilds.add(newValue); + } + } + return valueDesc; + } + + @VisibleForTesting + protected T interpretNodeAs(PrimitiveTypeInfo colTypeInfo, T constChild) { + if (exprFactory.isConstantExpr(constChild)) { + // Try to narrow type of constant + Object constVal = exprFactory.getConstantValue(constChild); + if (constVal == null) { + // adjust type of null + return exprFactory.createConstantExpr(colTypeInfo, null); + } + Object newConst = exprFactory.interpretConstantAsPrimitive( + colTypeInfo, constVal, (PrimitiveTypeInfo) exprFactory.getTypeInfo(constChild)); + if (newConst == null) { + return null; + } + if (newConst == constVal) { + return constChild; + } else { + return exprFactory.createConstantExpr(exprFactory.adjustConstantType(colTypeInfo, newConst), newConst); + } + } + return constChild; + } + + /** + * Returns true if des is a descendant of ans (ancestor) + */ + private boolean isDescendant(Node ans, Node des) { + if (ans.getChildren() == null) { + return false; + } + for (Node c : ans.getChildren()) { + if (c == des) { + return true; + } + if (isDescendant(c, des)) { + return true; + } + } + return false; + } + + protected T processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr, + Object... nodeOutputs) throws SemanticException { + RowResolver input = ctx.getInputRR(); + String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0) + .getText()); + // NOTE: tableAlias must be a valid non-ambiguous table alias, + // because we've checked that in TOK_TABLE_OR_COL's process method. + T desc = (T) nodeOutputs[1]; + String colName; + if (exprFactory.isConstantExpr(desc)) { + colName = exprFactory.getConstantValue(desc).toString(); + } else if (exprFactory.isColumnRefExpr(desc)) { + colName = exprFactory.getColumnName(desc); + } else { + throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]); + } + ColumnInfo colInfo = input.get(tableAlias, colName); + + // Try outer Row resolver + if (colInfo == null && ctx.getOuterRR() != null) { + RowResolver outerRR = ctx.getOuterRR(); + colInfo = outerRR.get(tableAlias, colName); + } + + if (colInfo == null) { + ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr); + return null; + } + return exprFactory.toExpr(colInfo); + } + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + // Here we know nd represents a group by expression. + + // During the DFS traversal of the AST, a descendant of nd likely set an + // error because a sub-tree of nd is unlikely to also be a group by + // expression. For example, in a query such as + // SELECT *concat(key)* FROM src GROUP BY concat(key), 'key' will be + // processed before 'concat(key)' and since 'key' is not a group by + // expression, an error will be set in ctx by ColumnExprProcessor. + + // We can clear the global error when we see that it was set in a + // descendant node of a group by expression because + // processGByExpr() returns a ExprNodeDesc that effectively ignores + // its children. Although the error can be set multiple times by + // descendant nodes, DFS traversal ensures that the error only needs to + // be cleared once. Also, for a case like + // SELECT concat(value, concat(value))... the logic still works as the + // error is only set with the first 'value'; all node processors quit + // early if the global error is set. + + if (isDescendant(nd, ctx.getErrorSrcNode())) { + ctx.setError(null, null); + } + return desc; + } + + if (ctx.getError() != null) { + return null; + } + + ASTNode expr = (ASTNode) nd; + + /* + * A Windowing specification get added as a child to a UDAF invocation to distinguish it + * from similar UDAFs but on different windows. + * The UDAF is translated to a WindowFunction invocation in the PTFTranslator. + * So here we just return null for tokens that appear in a Window Specification. + * When the traversal reaches up to the UDAF invocation its ExprNodeDesc is build using the + * ColumnInfo in the InputRR. This is similar to how UDAFs are handled in Select lists. + * The difference is that there is translation for Window related tokens, so we just + * return null; + */ + if (WINDOWING_TOKENS.contains(expr.getType())) { + if (!ctx.getallowWindowing()) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, + ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context"))); + } + + return null; + } + + if (expr.getType() == HiveParser.TOK_SUBQUERY_OP || expr.getType() == HiveParser.TOK_QUERY) { + return null; + } + + if (expr.getType() == HiveParser.TOK_TABNAME) { + return null; + } + + if (expr.getType() == HiveParser.TOK_ALLCOLREF) { + if (!ctx.getallowAllColRef()) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, + ErrorMsg.INVALID_COLUMN + .getMsg("All column reference is not supported in the context"))); + } + + RowResolver input = ctx.getInputRR(); + T columnList = exprFactory.createExprsListExpr(); + assert expr.getChildCount() <= 1; + if (expr.getChildCount() == 1) { + // table aliased (select a.*, for example) + ASTNode child = (ASTNode) expr.getChild(0); + assert child.getType() == HiveParser.TOK_TABNAME; + assert child.getChildCount() == 1; + String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText()); + Map columns = input.getFieldMap(tableAlias); + if (columns == null) { + throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(child)); + } + for (Map.Entry colMap : columns.entrySet()) { + ColumnInfo colInfo = colMap.getValue(); + if (!colInfo.getIsVirtualCol()) { + columnList = exprFactory.addExprToExprsList(columnList, exprFactory.toExpr(colInfo)); + } + } + } else { + // all columns (select *, for example) + for (ColumnInfo colInfo : input.getColumnInfos()) { + if (!colInfo.getIsVirtualCol()) { + columnList = exprFactory.addExprToExprsList(columnList, exprFactory.toExpr(colInfo)); + } + } + } + return columnList; + } + + // If the first child is a TOK_TABLE_OR_COL, and nodeOutput[0] is NULL, + // and the operator is a DOT, then it's a table column reference. + if (expr.getType() == HiveParser.DOT + && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL + && nodeOutputs[0] == null) { + return processQualifiedColRef(ctx, expr, nodeOutputs); + } + + // Return nulls for conversion operators + if (CONVERSION_FUNCTION_TEXT_MAP.keySet().contains(expr.getType()) + || expr.getToken().getType() == HiveParser.CharSetName + || expr.getToken().getType() == HiveParser.CharSetLiteral) { + return null; + } + + boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION || + expr.getType() == HiveParser.TOK_FUNCTIONSTAR || + expr.getType() == HiveParser.TOK_FUNCTIONDI); + + if (!ctx.getAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) { + throw new SemanticException( + SemanticAnalyzer.generateErrorMessage(expr, ErrorMsg.DISTINCT_NOT_SUPPORTED.getMsg())); + } + + // Create all children + int childrenBegin = (isFunction ? 1 : 0); + List children = new ArrayList( + expr.getChildCount() - childrenBegin); + for (int ci = childrenBegin; ci < expr.getChildCount(); ci++) { + T nodeOutput = (T) nodeOutputs[ci]; + if (exprFactory.isExprsListExpr(nodeOutput)) { + children.addAll(exprFactory.getExprChildren(nodeOutput)); + } else { + children.add(nodeOutput); + } + } + + if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) { + if (!ctx.getallowFunctionStar()) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr, + ErrorMsg.INVALID_COLUMN + .getMsg(".* reference is not supported in the context"))); + } + + RowResolver input = ctx.getInputRR(); + for (ColumnInfo colInfo : input.getColumnInfos()) { + if (!colInfo.getIsVirtualCol()) { + children.add(exprFactory.toExpr(colInfo)); + } + } + } + + // If any of the children contains null, then return a null + // this is a hack for now to handle the group by case + if (children.contains(null)) { + List possibleColumnNames = getReferenceableColumnAliases(ctx); + String reason = String.format("(possible column names are: %s)", + StringUtils.join(possibleColumnNames, ", ")); + ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(0), reason), + expr); + return null; + } + + // Create function desc + try { + return getXpathOrFuncExprNodeDesc(expr, isFunction, children, ctx); + } catch (UDFArgumentTypeException e) { + throw new SemanticException(ErrorMsg.INVALID_ARGUMENT_TYPE.getMsg(expr + .getChild(childrenBegin + e.getArgumentId()), e.getMessage()), e); + } catch (UDFArgumentLengthException e) { + throw new SemanticException(ErrorMsg.INVALID_ARGUMENT_LENGTH.getMsg( + expr, e.getMessage()), e); + } catch (UDFArgumentException e) { + throw new SemanticException(ErrorMsg.INVALID_ARGUMENT.getMsg(expr, e + .getMessage()), e); + } + } + + protected List getReferenceableColumnAliases(TypeCheckCtx ctx) { + return ctx.getInputRR().getReferenceableColumnAliases(null, -1); + } + } + + /** + * Factory method to get DefaultExprProcessor. + * + * @return DefaultExprProcessor. + */ + protected DefaultExprProcessor getDefaultExprProcessor() { + return new DefaultExprProcessor(); + } + + /** + * Processor for subquery expressions.. + */ + public class SubQueryExprProcessor implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + if (ctx.getError() != null) { + return null; + } + + ASTNode expr = (ASTNode) nd; + ASTNode sqNode = (ASTNode) expr.getParent().getChild(1); + + if (!ctx.getallowSubQueryExpr()) { + throw new CalciteSubquerySemanticException(SemanticAnalyzer.generateErrorMessage(sqNode, + ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg("Currently SubQuery expressions are only allowed as " + + "Where and Having Clause predicates"))); + } + + T desc = processGByExpr(nd, procCtx); + if (desc != null) { + return desc; + } + + //TOK_SUBQUERY_EXPR should have either 2 or 3 children + assert (expr.getChildren().size() == 3 || expr.getChildren().size() == 2); + //First child should be operand + assert (expr.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP); + + ASTNode subqueryOp = (ASTNode) expr.getChild(0); + SubqueryType subqueryType = null; + if ((subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_IN + || subqueryOp.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP_NOTIN)) { + subqueryType = SubqueryType.IN; + } else if ((subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_EXISTS + || subqueryOp.getChild(0).getType() == HiveParser.TOK_SUBQUERY_OP_NOTEXISTS)) { + subqueryType = SubqueryType.EXISTS; + } else if ((subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_SOME)) { + subqueryType = SubqueryType.SOME; + } else if ((subqueryOp.getChildCount() > 0) && (subqueryOp.getChild(0).getType() == HiveParser.KW_ALL)) { + subqueryType = SubqueryType.ALL; + } else if (subqueryOp.getChildCount() == 0) { + subqueryType = SubqueryType.SCALAR; + } + + T res = exprFactory.createSubqueryExpr(ctx, expr, subqueryType, (T[]) nodeOutputs); + if (res == null) { + /* + * Restriction.1.h :: SubQueries only supported in the SQL Where Clause. + */ + ctx.setError(ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg(sqNode, + "Currently only IN & EXISTS SubQuery expressions are allowed"), + sqNode); + } + return expr; + } + } + + /** + * Factory method to get SubQueryExprProcessor. + * + * @return DateExprProcessor. + */ + protected SubQueryExprProcessor getSubQueryExprProcessor() { + return new SubQueryExprProcessor(); + } + + /** + * Function to do groupby subexpression elimination. This is called by all the + * processors initially. As an example, consider the query select a+b, + * count(1) from T group by a+b; Then a+b is already precomputed in the group + * by operators key, so we substitute a+b in the select list with the internal + * column name of the a+b expression that appears in the in input row + * resolver. + * + * @param nd The node that is being inspected. + * @param procCtx The processor context. + * @return exprNodeColumnDesc. + */ + private T processGByExpr(Node nd, Object procCtx) throws SemanticException { + // We recursively create the exprNodeDesc. Base cases: when we encounter + // a column ref, we convert that into an exprNodeColumnDesc; when we + // encounter + // a constant, we convert that into an exprNodeConstantDesc. For others we + // just + // build the exprNodeFuncDesc with recursively built children. + ASTNode expr = (ASTNode) nd; + TypeCheckCtx ctx = (TypeCheckCtx) procCtx; + + // bypass only if outerRR is not null. Otherwise we need to look for expressions in outerRR for + // subqueries e.g. select min(b.value) from table b group by b.key + // having key in (select .. where a = min(b.value) + if (!ctx.isUseCaching() && ctx.getOuterRR() == null) { + return null; + } + + RowResolver input = ctx.getInputRR(); + T desc = null; + + if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) { + return null; + } + + // If the current subExpression is pre-calculated, as in Group-By etc. + ColumnInfo colInfo = input.getExpression(expr); + + // try outer row resolver + RowResolver outerRR = ctx.getOuterRR(); + if (colInfo == null && outerRR != null) { + colInfo = outerRR.getExpression(expr); + } + if (colInfo != null) { + desc = exprFactory.createColumnRefExpr(colInfo); + ASTNode source = input.getExpressionSource(expr); + if (source != null && ctx.getUnparseTranslator() != null) { + ctx.getUnparseTranslator().addCopyTranslation(expr, source); + } + return desc; + } + return desc; + } + + public static boolean isStringType(TypeInfo typeInfo) { + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); + if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(primitiveCategory) == + PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP) { + return true; + } + } + return false; + } + + public static String getFunctionText(ASTNode expr, boolean isFunction) { + String funcText = null; + if (!isFunction) { + // For operator, the function name is the operator text, unless it's in + // our special dictionary + if (expr.getChildCount() == 1) { + funcText = SPECIAL_UNARY_OPERATOR_TEXT_MAP.get(expr.getType()); + } + if (funcText == null) { + funcText = expr.getText(); + } + } else { + // For TOK_FUNCTION, the function name is stored in the first child, + // unless it's in our + // special dictionary. + assert (expr.getChildCount() >= 1); + int funcType = ((ASTNode) expr.getChild(0)).getType(); + if (funcText == null) { + funcText = CONVERSION_FUNCTION_TEXT_MAP.get(funcType); + } + if (funcText == null) { + funcText = ((ASTNode) expr.getChild(0)).getText(); + } + } + return BaseSemanticAnalyzer.unescapeIdentifier(funcText); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactoryUtils.java similarity index 96% rename from ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java rename to ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactoryUtils.java index b0544f3d6f..f1c9850a7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactoryUtils.java @@ -16,13 +16,14 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.parse.type; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -39,7 +40,7 @@ public class TypeCheckProcFactoryUtils { - static ArrayList rewriteInToOR(ArrayList inOperands) throws SemanticException { + static List rewriteInToOR(List inOperands) throws SemanticException { ExprNodeDesc columnDesc = inOperands.get(0); ArrayList orOperands = new ArrayList<>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index d887ca8015..daea26d6d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -20,6 +20,7 @@ import com.google.common.collect.Multimap; import java.util.Collection; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -1001,17 +1002,10 @@ public static boolean isSame(List first, List second // Given an expression this method figures out if the type for the expression belongs to string group // e.g. (String, Char, Varchar etc) public static boolean isStringType(ExprNodeDesc expr) { - TypeInfo typeInfo = expr.getTypeInfo(); - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { - PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); - if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(primitiveCategory) == - PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP) { - return true; - } - } - return false; + return TypeCheckProcFactory.isStringType(expr.getTypeInfo()); } - // Given an expression this method figures out if the type for the expression is integer + + // Given an expression this method figures out if the type for the expression is integer // i.e. INT, SHORT, TINYINT (BYTE) or LONG public static boolean isIntegerType(ExprNodeDesc expr) { TypeInfo typeInfo = expr.getTypeInfo(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java index 002aef6ad2..e4e430b7e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java @@ -226,8 +226,7 @@ public ExprNodeDesc clone() { * @throws UDFArgumentException */ public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF, - String funcText, - List children) throws UDFArgumentException { + String funcText, List children) throws UDFArgumentException { ObjectInspector[] childrenOIs = new ObjectInspector[children.size()]; for (int i = 0; i < childrenOIs.length; i++) { childrenOIs[i] = children.get(i).getWritableObjectInspector(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeSubQueryDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeSubQueryDesc.java index cd80da88f1..22b7152b54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeSubQueryDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeSubQueryDesc.java @@ -33,14 +33,6 @@ public class ExprNodeSubQueryDesc extends ExprNodeDesc implements Serializable { private static final long serialVersionUID = 1L; - public static enum SubqueryType{ - IN, - EXISTS, - SCALAR, - SOME, - ALL - }; - /** * RexNode corresponding to subquery. */ @@ -65,7 +57,7 @@ public ExprNodeSubQueryDesc(TypeInfo typeInfo, RelNode subQuery, this.comparisonOp = null; } public ExprNodeSubQueryDesc(TypeInfo typeInfo, RelNode subQuery, - SubqueryType type, ExprNodeDesc lhs, ASTNode comparisonOp) { + SubqueryType type, ExprNodeDesc lhs, ASTNode comparisonOp) { super(typeInfo); this.rexSubQuery = subQuery; this.subQueryLhs = lhs; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 2314f49631..894a93c449 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -63,7 +63,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; @@ -863,7 +863,8 @@ public static ReduceSinkDesc getReduceSinkDesc( partitionCols.addAll(keyCols.subList(0, numPartitionFields)); } else { // numPartitionFields = -1 means random partitioning - partitionCols.add(TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("rand")); + partitionCols.add(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(). + getFuncExprNodeDesc("rand")); } StringBuilder order = new StringBuilder(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SubqueryType.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SubqueryType.java new file mode 100644 index 0000000000..017f9b5632 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SubqueryType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.plan; + +public enum SubqueryType { + IN, + EXISTS, + SCALAR, + SOME, + ALL +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java index 062be74c3a..657caa82f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java @@ -21,9 +21,8 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java index 6b37a59ab1..9e52be8f81 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java @@ -35,8 +35,8 @@ import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckCtx; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -815,7 +815,7 @@ public static ExprNodeDesc buildExprNode(ASTNode expr, { // todo: use SemanticAnalyzer::genExprNodeDesc // currently SA not available to PTFTranslator. - Map map = TypeCheckProcFactory + Map map = ExprNodeTypeCheck .genExprNode(expr, typeCheckCtx); ExprNodeDesc desc = map.get(expr); if (desc == null) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java index 34fe2b9dcf..b33ebd2c03 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -150,7 +150,7 @@ public void testExprNodeFuncEvaluator() throws Throwable { false); ExprNodeDesc col11desc = getListIndexNode(col1desc, 1); ExprNodeDesc cola0desc = getListIndexNode(coladesc, 0); - ExprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc func1 = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", col11desc, cola0desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); @@ -173,7 +173,7 @@ public void testExprNodeConversionEvaluator() throws Throwable { ExprNodeDesc col1desc = new ExprNodeColumnDesc(col1Type, "col1", "", false); ExprNodeDesc col11desc = getListIndexNode(col1desc, 1); - ExprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc func1 = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc(serdeConstants.DOUBLE_TYPE_NAME, col11desc); ExprNodeEvaluator eval = ExprNodeEvaluatorFactory.get(func1); @@ -215,22 +215,22 @@ public void testExprNodeSpeed() throws Throwable { try { int basetimes = 100000; measureSpeed("1 + 2", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc( + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor().getFuncExprNodeDesc( "+", new ExprNodeConstantDesc(1), new ExprNodeConstantDesc(2))), r, Integer.valueOf(1 + 2)); measureSpeed("1 + 2 - 3", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("-", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("+", new ExprNodeConstantDesc(1), new ExprNodeConstantDesc(2)), new ExprNodeConstantDesc(3))), r, Integer.valueOf(1 + 2 - 3)); measureSpeed("1 + 2 - 3 + 4", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("+", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("-", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("+", new ExprNodeConstantDesc(1), new ExprNodeConstantDesc(2)), @@ -239,25 +239,25 @@ public void testExprNodeSpeed() throws Throwable { .valueOf(1 + 2 - 3 + 4)); measureSpeed("concat(\"1\", \"2\")", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", new ExprNodeConstantDesc("1"), new ExprNodeConstantDesc("2"))), r, "12"); measureSpeed("concat(concat(\"1\", \"2\"), \"3\")", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", new ExprNodeConstantDesc("1"), new ExprNodeConstantDesc("2")), new ExprNodeConstantDesc("3"))), r, "123"); measureSpeed("concat(concat(concat(\"1\", \"2\"), \"3\"), \"4\")", basetimes * 100, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", new ExprNodeConstantDesc("1"), new ExprNodeConstantDesc("2")), @@ -267,16 +267,16 @@ public void testExprNodeSpeed() throws Throwable { ExprNodeDesc constant2 = new ExprNodeConstantDesc(2); measureSpeed("concat(col1[1], cola[1])", basetimes * 10, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", getListIndexNode( new ExprNodeColumnDesc(col1Type, "col1", "", false), constant1), getListIndexNode(new ExprNodeColumnDesc( colaType, "cola", "", false), constant1))), r, "1b"); measureSpeed("concat(concat(col1[1], cola[1]), col1[2])", basetimes * 10, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", getListIndexNode( new ExprNodeColumnDesc(col1Type, "col1", "", false), constant1), getListIndexNode( @@ -287,11 +287,11 @@ public void testExprNodeSpeed() throws Throwable { measureSpeed( "concat(concat(concat(col1[1], cola[1]), col1[2]), cola[2])", basetimes * 10, ExprNodeEvaluatorFactory - .get(TypeCheckProcFactory.DefaultExprProcessor + .get(ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", - TypeCheckProcFactory.DefaultExprProcessor + ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", getListIndexNode(new ExprNodeColumnDesc( col1Type, "col1", "", false), diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index dc00ceb398..8a0606b7a3 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -34,7 +34,8 @@ import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin; import org.apache.hadoop.hive.ql.optimizer.physical.LlapClusterStateForCompile; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.CollectDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -232,7 +233,7 @@ public void testScriptOperator() throws Throwable { ExprNodeDesc expr1 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "col0", "", false); ExprNodeDesc expr2 = new ExprNodeConstantDesc("1"); - ExprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc exprDesc2 = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("concat", expr1, expr2); // select operator to project these two columns diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java index ca227346f4..0968a438f7 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; @@ -60,7 +60,7 @@ public void testPlan() throws Exception { TypeInfoFactory.stringTypeInfo, f1, "", false); ExprNodeDesc expr2 = new ExprNodeColumnDesc( TypeInfoFactory.stringTypeInfo, f2, "", false); - ExprNodeDesc filterExpr = TypeCheckProcFactory.DefaultExprProcessor + ExprNodeDesc filterExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() .getFuncExprNodeDesc("==", expr1, expr2); FilterDesc filterCtx = new FilterDesc(filterExpr, false); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestTypeCheckProcFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestTypeCheckProcFactory.java similarity index 85% rename from ql/src/test/org/apache/hadoop/hive/ql/parse/TestTypeCheckProcFactory.java rename to ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestTypeCheckProcFactory.java index 66d024a162..b27dacb4b4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestTypeCheckProcFactory.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestTypeCheckProcFactory.java @@ -15,28 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.mockito.Mockito.when; +package org.apache.hadoop.hive.ql.parse.type; import java.math.BigDecimal; import java.util.Arrays; import java.util.Collection; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory.DefaultExprProcessor; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory.DefaultExprProcessor; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; /** @@ -78,26 +75,26 @@ public TestTypeCheckProcFactory(String maxValue, PrimitiveTypeEntry constType, O @Before public void init() { MockitoAnnotations.initMocks(this); - testSubject = new DefaultExprProcessor(); + testSubject = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(); } public void testOneCase(Object constValue) { - when(nodeDesc.getValue()).thenReturn(constValue); - when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType); + Mockito.when(nodeDesc.getValue()).thenReturn(constValue); + Mockito.when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType); ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAs(typeInfo, nodeDesc); - assertNotNull(result); - assertEquals(expectedValue, result.getValue()); + Assert.assertNotNull(result); + Assert.assertEquals(expectedValue, result.getValue()); } public void testNullCase(Object constValue) { - when(nodeDesc.getValue()).thenReturn(constValue); - when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType); + Mockito.when(nodeDesc.getValue()).thenReturn(constValue); + Mockito.when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType); ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAs(typeInfo, nodeDesc); - assertNull(result); + Assert.assertNull(result); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFConcat.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFConcat.java index 11ba426769..8519a1265a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFConcat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFConcat.java @@ -22,7 +22,8 @@ import java.util.List; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.testutil.BaseScalarUdfTest; import org.apache.hadoop.hive.ql.testutil.DataBuilder; @@ -65,7 +66,8 @@ public List getExpressionList() throws UDFArgumentException { ExprNodeDesc expr1 = OperatorTestUtils.getStringColumn("a"); ExprNodeDesc expr2 = OperatorTestUtils.getStringColumn("b"); - ExprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", expr1, expr2); + ExprNodeDesc exprDesc2 = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("concat", expr1, expr2); List earr = new ArrayList(); earr.add(expr1); earr.add(exprDesc2); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRound.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRound.java index c1beed6fec..8c871d5500 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRound.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRound.java @@ -21,11 +21,12 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; import org.junit.Assert; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.testutil.BaseScalarUdfTest; @@ -110,7 +111,8 @@ List earr = new ArrayList(); for (int j = 0; j < cols.length; j++) { - ExprNodeDesc r = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("round", exprs.get(j), scales[j]); + ExprNodeDesc r = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .getFuncExprNodeDesc("round", exprs.get(j), scales[j]); earr.add(r); }