diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java index dccd1d9..9c5163e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.JoinType; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -51,9 +52,9 @@ private final ImmutableList inputRRLst; private final boolean outerJoin; - public JoinTypeCheckCtx(RowResolver leftRR, RowResolver rightRR, JoinType hiveJoinType) + public JoinTypeCheckCtx(Configuration conf, RowResolver leftRR, RowResolver rightRR, JoinType hiveJoinType) throws SemanticException { - super(RowResolver.getCombinedRR(leftRR, rightRR), true, false, false, false, false, false, false, + super(conf, RowResolver.getCombinedRR(leftRR, rightRR), true, false, false, false, false, false, false, false, false); this.inputRRLst = ImmutableList.of(leftRR, rightRR); this.outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index fbe93f9..2688320 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1234,7 +1234,7 @@ private static boolean getPartExprNodeDesc(ASTNode astNode, HiveConf conf, return astNode.getType() != HiveParser.TOK_PARTVAL; } - TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null); + TypeCheckCtx typeCheckCtx = new TypeCheckCtx(conf, null); String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME); boolean result = true; for (Node childNode : astNode.getChildren()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 61ee2bd..56a303a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1216,7 +1216,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ // 2. Construct ExpressionNodeDesc representing Join Condition RexNode calciteJoinCond = null; if (joinCond != null) { - JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); + JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(conf, leftRR, rightRR, hiveJoinType); Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, jCtx); if (jCtx.getError() != null) @@ -1949,7 +1949,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver // 3.4 Try GenericUDF translation if (udafRetType == null) { - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); // We allow stateful functions in the SELECT list (but nowhere else) tcCtx.setAllowStatefulFunctions(true); tcCtx.setAllowDistinctFunctions(false); @@ -2046,7 +2046,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException for (int i = 0; i < grpByAstExprs.size(); ++i) { ASTNode grpbyExpr = grpByAstExprs.get(i); Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); + grpbyExpr, new TypeCheckCtx(conf, groupByInputRowResolver)); ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); if (grpbyExprNDesc == null) throw new CalciteSemanticException("Invalid Column Reference: " + grpbyExpr.dump(), @@ -2192,7 +2192,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 2.1 Convert AST Expr to ExprNode obASTExpr = (ASTNode) obASTExprLst.get(i); Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - obASTExpr, new TypeCheckCtx(inputRR)); + obASTExpr, new TypeCheckCtx(conf, inputRR)); ExprNodeDesc obExprNDesc = astToExprNDescMap.get(obASTExpr.getChild(0)); if (obExprNDesc == null) throw new SemanticException("Invalid order by expression: " + obASTExpr.toString()); @@ -2324,7 +2324,7 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept if (ps != null) { List pExprs = ps.getExpressions(); for (PartitionExpression pExpr : pExprs) { - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); tcCtx.setAllowStatefulFunctions(true); ExprNodeDesc exp = genExprNodeDesc(pExpr.getExpression(), inputRR, tcCtx); pKeys.add(converter.convert(exp)); @@ -2340,7 +2340,7 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept if (os != null) { List oExprs = os.getExpressions(); for (OrderExpression oExpr : oExprs) { - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); tcCtx.setAllowStatefulFunctions(true); ExprNodeDesc exp = genExprNodeDesc(oExpr.getExpression(), inputRR, tcCtx); RexNode ordExp = converter.convert(exp); @@ -2711,7 +2711,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) UnsupportedFeature.Distinct_without_an_aggreggation); } else { // Case when this is an expression - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); // We allow stateful functions in the SELECT list (but nowhere else) tcCtx.setAllowStatefulFunctions(true); ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 2d7d9d4..0f2de86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -2980,7 +2980,7 @@ private void analyzeMetastoreCheck(CommonTree ast) throws SemanticException { String key = stripIdentifierQuotes(partSpecSingleKey.getChild(0).getText()).toLowerCase(); String operator = partSpecSingleKey.getChild(1).getText(); ASTNode partValNode = (ASTNode)partSpecSingleKey.getChild(2); - TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null); + TypeCheckCtx typeCheckCtx = new TypeCheckCtx(conf, null); ExprNodeConstantDesc valExpr = (ExprNodeConstantDesc)TypeCheckProcFactory .genExprNode(partValNode, typeCheckCtx).get(partValNode); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java index e0cd398..662a71e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java @@ -651,7 +651,7 @@ private ShapeDetails setupShape(StructObjectInspector OI, shp.setSerdeProps(serdePropsMap); shp.setColumnNames(columnNames); - TypeCheckCtx tCtx = new TypeCheckCtx(rr); + TypeCheckCtx tCtx = new TypeCheckCtx(hCfg, rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); @@ -684,7 +684,7 @@ private ShapeDetails setupShapeForNoop(ShapeDetails inpShape, shp.setSerdeProps(inpShape.getSerdeProps()); shp.setColumnNames(columnNames); - TypeCheckCtx tCtx = new TypeCheckCtx(rr); + TypeCheckCtx tCtx = new TypeCheckCtx(hCfg, rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java index f95ee8d..f333fdc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java @@ -325,11 +325,11 @@ Conjunct analyzeConjunct(ASTNode conjunct) throws SemanticException { */ protected ColumnInfo resolveDot(ASTNode node) { try { - TypeCheckCtx tcCtx = new TypeCheckCtx(parentQueryRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(ctx.getConf(), parentQueryRR); String str = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(1).getText()); ExprNodeDesc idDesc = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, str); ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) - defaultExprProcessor.process(node, stack, tcCtx, (Object) null, idDesc); + defaultExprProcessor.process(node, stack, tcCtx, null, idDesc); if ( colDesc != null ) { String[] qualName = parentQueryRR.reverseLookup(colDesc.getColumn()); return parentQueryRR.get(qualName[0], qualName[1]); @@ -460,6 +460,8 @@ void setSQRR(RowResolver sqRR) { private QBSubQueryRewrite subQueryDiagnostic; + private Context ctx; + public QBSubQuery(String outerQueryId, int sqIdx, ASTNode subQueryAST, @@ -476,6 +478,7 @@ public QBSubQuery(String outerQueryId, this.alias = "sq_" + this.sqIdx; this.numCorrExprsinSQ = 0; this.numOuterCorrExprsForHaving = 0; + this.ctx = ctx; String s = ctx.getTokenRewriteStream().toString( originalSQAST.getTokenStartIndex(), originalSQAST.getTokenStopIndex()); originalSQASTOrigin = new ASTNodeOrigin("SubQuery", alias, s, alias, originalSQAST); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4af07ad..eb0c18e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3808,7 +3808,7 @@ static boolean isRegex(String pattern, HiveConf conf) { expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); // We allow stateful functions in the SELECT list (but nowhere else) tcCtx.setAllowStatefulFunctions(true); tcCtx.setAllowDistinctFunctions(false); @@ -3947,7 +3947,7 @@ public RowResolver handleInsertStatementSpec(List col_list, String } else { //add new 'synthetic' columns for projections not provided by Select - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, inputRR); CommonToken t = new CommonToken(HiveParser.TOK_NULL); t.setText("TOK_NULL"); ExprNodeDesc exp = genExprNodeDesc(new ASTNode(t), inputRR, tcCtx); @@ -10438,7 +10438,7 @@ public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input) public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching) throws SemanticException { - TypeCheckCtx tcCtx = new TypeCheckCtx(input, useCaching); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, input, useCaching); return genExprNodeDesc(expr, input, tcCtx); } @@ -10448,7 +10448,7 @@ public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean use */ public Map genAllExprNodeDesc(ASTNode expr, RowResolver input) throws SemanticException { - TypeCheckCtx tcCtx = new TypeCheckCtx(input); + TypeCheckCtx tcCtx = new TypeCheckCtx(conf, input); return genAllExprNodeDesc(expr, input, tcCtx); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java index 8ad28be..bd79c9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java @@ -20,6 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; /** @@ -59,6 +60,8 @@ private boolean allowDistinctFunctions; + private final Configuration conf; + private final boolean allowGBExprElimination; private final boolean allowAllColRef; @@ -78,20 +81,21 @@ * @param inputRR * The input row resolver of the previous operator. */ - public TypeCheckCtx(RowResolver inputRR) { - this(inputRR, true); + public TypeCheckCtx(Configuration conf, RowResolver inputRR) { + this(conf, inputRR, true); } - public TypeCheckCtx(RowResolver inputRR, boolean useCaching) { - this(inputRR, useCaching, false, true, true, true, true, true, true, true); + public TypeCheckCtx(Configuration conf, RowResolver inputRR, boolean useCaching) { + this(conf, inputRR, useCaching, false, true, true, true, true, true, true, true); } - public TypeCheckCtx(RowResolver inputRR, boolean useCaching, boolean allowStatefulFunctions, + public TypeCheckCtx(Configuration conf, RowResolver inputRR, boolean useCaching, boolean allowStatefulFunctions, boolean allowDistinctFunctions, boolean allowGBExprElimination, boolean allowAllColRef, boolean allowFunctionStar, boolean allowWindowing, boolean allowIndexExpr, boolean allowSubQueryExpr) { setInputRR(inputRR); error = null; + this.conf = conf; this.useCaching = useCaching; this.allowStatefulFunctions = allowStatefulFunctions; this.allowDistinctFunctions = allowDistinctFunctions; @@ -209,4 +213,8 @@ public boolean getallowSubQueryExpr() { public boolean isUseCaching() { return useCaching; } + + public Configuration getConfig() { + return conf; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index ab5d006..0b65117 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -133,7 +133,7 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) RowResolver input = ctx.getInputRR(); ExprNodeDesc desc = null; - if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) { + if (input == null || !ctx.getAllowGBExprElimination()) { return null; } @@ -210,15 +210,20 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) HashMap nodeOutputs = new LinkedHashMap(); ogw.startWalking(topNodes, nodeOutputs); - return convert(nodeOutputs); + return convert(nodeOutputs, tcCtx); } // temporary type-safe casting - private static Map convert(Map outputs) { + private static Map convert(Map outputs, TypeCheckCtx ctx) + throws SemanticException { Map converted = new LinkedHashMap(); for (Map.Entry entry : outputs.entrySet()) { if (entry.getKey() instanceof ASTNode && (entry.getValue() == null || entry.getValue() instanceof ExprNodeDesc)) { + ExprNodeDesc nodeDsc = (ExprNodeDesc)entry.getValue(); + if (nodeDsc != null) { + nodeDsc.validate(ctx.getConfig()); + } converted.put((ASTNode)entry.getKey(), (ExprNodeDesc)entry.getValue()); } else { LOG.warn("Invalid type entry " + entry); @@ -836,7 +841,7 @@ protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, F if (fi.getGenericUDTF() != null) { throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg()); } - // UDAF in filter condition, group-by caluse, param of funtion, etc. + // UDAF in filter condition, group-by clause, param of function, etc. if (fi.getGenericUDAFResolver() != null) { if (isFunction) { throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg((ASTNode) expr diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnListDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnListDesc.java index be78e73..8d60cc9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnListDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnListDesc.java @@ -63,6 +63,18 @@ public void setTypeInfo(TypeInfo typeInfo) { } @Override + public String getExprString() { + StringBuilder builder = new StringBuilder(); + for (ExprNodeDesc expr : columns) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expr.getExprString()); + } + return "[" + builder.toString() + "]"; + } + + @Override public ObjectInspector getWritableObjectInspector() { throw new IllegalStateException(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java index 328bd86..b334cad 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java @@ -23,6 +23,8 @@ import java.util.Collection; import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -76,10 +78,7 @@ public void setTypeInfo(TypeInfo typeInfo) { this.typeInfo = typeInfo; } - public String getExprString() { - assert (false); - return null; - } + public abstract String getExprString(); public ObjectInspector getWritableObjectInspector() { return TypeInfoUtils @@ -104,6 +103,9 @@ public String getName() { return this.getClass().getName(); } + public void validate(Configuration conf) throws UDFArgumentException { + } + // This wraps an instance of an ExprNodeDesc, and makes equals work like isSame, see comment on // isSame public final static class ExprNodeDescEqualityWrapper { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java index b5d2ddf..6c920f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java @@ -62,7 +62,7 @@ * exactly what we want. */ private GenericUDF genericUDF; - private List chidren; + private List children; private transient String funcText; /** * This class uses a writableObjectInspector rather than a TypeInfo to store @@ -72,7 +72,7 @@ //Is this an expression that should perform a comparison for sorted searches private boolean isSortedExpr; - public ExprNodeGenericFuncDesc() {; + public ExprNodeGenericFuncDesc() { } /* If the function has an explicit name like func(args) then call a @@ -94,7 +94,7 @@ public ExprNodeGenericFuncDesc(ObjectInspector oi, GenericUDF genericUDF, ObjectInspectorUtils.getWritableObjectInspector(oi); assert (genericUDF != null); this.genericUDF = genericUDF; - this.chidren = children; + this.children = children; this.funcText = funcText; } @@ -123,12 +123,12 @@ public void setGenericUDF(GenericUDF genericUDF) { } public void setChildren(List children) { - chidren = children; + this.children = children; } @Override public List getChildren() { - return chidren; + return children; } @Override @@ -136,12 +136,12 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append(genericUDF.getClass().getSimpleName()); sb.append("("); - if (chidren != null) { - for (int i = 0; i < chidren.size(); i++) { + if (children != null) { + for (int i = 0; i < children.size(); i++) { if (i > 0) { sb.append(", "); } - sb.append(chidren.get(i)); + sb.append(children.get(i)); } } sb.append(")"); @@ -151,9 +151,9 @@ public String toString() { @Override public String getExprString() { // Get the children expr strings - String[] childrenExprStrings = new String[chidren.size()]; + String[] childrenExprStrings = new String[children.size()]; for (int i = 0; i < childrenExprStrings.length; i++) { - childrenExprStrings[i] = chidren.get(i).getExprString(); + childrenExprStrings[i] = children.get(i).getExprString(); } return genericUDF.getDisplayString(childrenExprStrings); @@ -162,10 +162,10 @@ public String getExprString() { @Override public List getCols() { List colList = new ArrayList(); - if (chidren != null) { + if (children != null) { int pos = 0; - while (pos < chidren.size()) { - List colCh = chidren.get(pos).getCols(); + while (pos < children.size()) { + List colCh = children.get(pos).getCols(); colList = Utilities.mergeUniqElems(colList, colCh); pos++; } @@ -176,8 +176,8 @@ public String getExprString() { @Override public ExprNodeDesc clone() { - List cloneCh = new ArrayList(chidren.size()); - for (ExprNodeDesc ch : chidren) { + List cloneCh = new ArrayList(children.size()); + for (ExprNodeDesc ch : children) { cloneCh.add(ch.clone()); } ExprNodeGenericFuncDesc clone = new ExprNodeGenericFuncDesc(typeInfo, @@ -201,36 +201,6 @@ public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF, childrenOIs[i] = children.get(i).getWritableObjectInspector(); } - // Check if a bigint is implicitely cast to a double as part of a comparison - // Perform the check here instead of in GenericUDFBaseCompare to guarantee it is only run once per operator - if (genericUDF instanceof GenericUDFBaseCompare && children.size() == 2) { - - TypeInfo oiTypeInfo0 = children.get(0).getTypeInfo(); - TypeInfo oiTypeInfo1 = children.get(1).getTypeInfo(); - - SessionState ss = SessionState.get(); - Configuration conf = (ss != null) ? ss.getConf() : new Configuration(); - - LogHelper console = new LogHelper(LOG); - - // For now, if a bigint is going to be cast to a double throw an error or warning - if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) || - (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo))) { - if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) { - throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_STRING.getMsg()); - } else { - console.printError("WARNING: Comparing a bigint and a string may result in a loss of precision."); - } - } else if ((oiTypeInfo0.equals(TypeInfoFactory.doubleTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) || - (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.doubleTypeInfo))) { - if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) { - throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_DOUBLE.getMsg()); - } else { - console.printError("WARNING: Comparing a bigint and a double may result in a loss of precision."); - } - } - } - ObjectInspector oi = genericUDF.initializeAndFoldConstants(childrenOIs); String[] requiredJars = genericUDF.getRequiredJars(); @@ -258,6 +228,36 @@ public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF, return new ExprNodeGenericFuncDesc(oi, genericUDF, funcText, children); } + @Override + public void validate(Configuration conf) throws UDFArgumentException { + // Check if a bigint is implicitly cast to a double as part of a comparison + // Perform the check here instead of in GenericUDFBaseCompare to guarantee it is only run once per operator + if (genericUDF instanceof GenericUDFBaseCompare && children.size() == 2) { + + TypeInfo oiTypeInfo0 = children.get(0).getTypeInfo(); + TypeInfo oiTypeInfo1 = children.get(1).getTypeInfo(); + + LogHelper console = new LogHelper(LOG); + + // For now, if a bigint is going to be cast to a double throw an error or warning + if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) || + (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo))) { + if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) { + throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_STRING.getMsg()); + } else { + console.printError("WARNING: Comparing a bigint and a string may result in a loss of precision."); + } + } else if ((oiTypeInfo0.equals(TypeInfoFactory.doubleTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) || + (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.doubleTypeInfo))) { + if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) { + throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_DOUBLE.getMsg()); + } else { + console.printError("WARNING: Comparing a bigint and a double may result in a loss of precision."); + } + } + } + } + /* Backward-compatibility interface for the case where there is no explicit * name for the function. */ @@ -287,12 +287,12 @@ public boolean isSame(Object o) { } } - if (chidren.size() != dest.getChildren().size()) { + if (children.size() != dest.getChildren().size()) { return false; } - for (int pos = 0; pos < chidren.size(); pos++) { - if (!chidren.get(pos).isSame(dest.getChildren().get(pos))) { + for (int pos = 0; pos < children.size(); pos++) { + if (!children.get(pos).isSame(dest.getChildren().get(pos))) { return false; } } @@ -305,7 +305,7 @@ public int hashCode() { int superHashCode = super.hashCode(); HashCodeBuilder builder = new HashCodeBuilder(); builder.appendSuper(superHashCode); - builder.append(chidren); + builder.append(children); return builder.toHashCode(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java index cc2b77b..01d881c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; @@ -186,7 +187,7 @@ public void setupOutputOI() throws SemanticException ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR); try { - resultExprParser.translate(); + resultExprParser.translate(cfg); } catch(HiveException he) { throw new SemanticException(he); @@ -215,8 +216,8 @@ public void setupOutputOI() throws SemanticException */ private void validateAndSetupPatternStr(MatchPath evaluator, List args) throws SemanticException { - PTFExpressionDef symboPatternArg = args.get(0); - ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); + PTFExpressionDef symbolPatternArg = args.get(0); + ObjectInspector symbolPatternArgOI = symbolPatternArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) || (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || @@ -693,7 +694,7 @@ else if ( isPlus ) * - the select keyword is optional. The parser checks if the expression doesn't start with * select; if not it prefixes it. * - Window Fn clauses are not permitted. - * - expressions can operate on the input columns plus the psuedo column 'path' + * - expressions can operate on the input columns plus the pseudo column 'path' * which is array of * structs. The shape of the struct is * the same as the input. @@ -716,9 +717,9 @@ public ResultExpressionParser(String resultExprString, this.selectListInputRowResolver = selectListInputRowResolver; } - public void translate() throws SemanticException, HiveException + public void translate(HiveConf conf) throws SemanticException, HiveException { - setupSelectListInputInfo(); + setupSelectListInputInfo(conf); fixResultExprString(); parse(); validateSelectExpr(); @@ -763,9 +764,9 @@ private void buildSelectListEvaluators() throws SemanticException, HiveException resultExprInfo.resultExprNames, selectListExprOIs); } - private void setupSelectListInputInfo() throws SemanticException + private void setupSelectListInputInfo(HiveConf conf) throws SemanticException { - selectListInputTypeCheckCtx = new TypeCheckCtx(selectListInputRowResolver); + selectListInputTypeCheckCtx = new TypeCheckCtx(conf, selectListInputRowResolver); selectListInputTypeCheckCtx.setUnparseTranslator(null); /* * create SelectListOI diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java index 71034d7..6dd3862 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java @@ -54,6 +54,7 @@ */ @SuppressWarnings("deprecation") public abstract class TableFunctionResolver { + HiveConf cfg; TableFunctionEvaluator evaluator; PTFDesc ptfDesc; @@ -65,6 +66,7 @@ */ public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) throws SemanticException { + this.cfg = cfg; this.ptfDesc = ptfDesc; evaluator = createEvaluator(ptfDesc, tDef); diff --git a/ql/src/test/results/clientnegative/compare_double_bigint.q.out b/ql/src/test/results/clientnegative/compare_double_bigint.q.out index 68a295a..47ad896 100644 --- a/ql/src/test/results/clientnegative/compare_double_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_double_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': In strict mode, comparing bigints and doubles is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict +FAILED: UDFArgumentException [Error 10124]: In strict mode, comparing bigints and doubles is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict diff --git a/ql/src/test/results/clientnegative/compare_string_bigint.q.out b/ql/src/test/results/clientnegative/compare_string_bigint.q.out index 9a604fa..0ecf43a 100644 --- a/ql/src/test/results/clientnegative/compare_string_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_string_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': In strict mode, comparing bigints and strings is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict +FAILED: UDFArgumentException [Error 10123]: In strict mode, comparing bigints and strings is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict