diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index f94a9f9d70..1cfd04e8e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3264,7 +3264,13 @@ private RelNode genFilterRelNode(ASTNode filterNode, RelNode srcRel, ImmutableMap outerNameToPosMap, RowResolver outerRR, boolean useCaching) throws SemanticException { RexNode filterExpression = genRexNode(filterNode, relToHiveRR.get(srcRel), - outerRR, null, useCaching, cluster.getRexBuilder()); + outerRR, null, useCaching, cluster.getRexBuilder()); + + return genFilterRelNode(filterExpression, srcRel, outerNameToPosMap, outerRR); + } + + private RelNode genFilterRelNode(RexNode filterExpression, RelNode srcRel, + ImmutableMap outerNameToPosMap, RowResolver outerRR) throws SemanticException { if (RexUtil.isLiteral(filterExpression, false) && filterExpression.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { // queries like select * from t1 where 'foo'; @@ -3584,6 +3590,35 @@ private RelNode genFilterLogicalPlan(QB qb, RelNode srcRel, ImmutableMap genConstraintFilterLogicalPlan( + QB qb, RelNode srcRel, ImmutableMap outerNameToPosMap, RowResolver outerRR) + throws SemanticException { + if (qb.getIsQuery()) { + return null; + } + + String dest = qb.getParseInfo().getClauseNames().iterator().next(); + if (!updating(dest)) { + return null; + } + + RowResolver inputRR = relToHiveRR.get(srcRel); + RexNode constraintUDF = RexNodeTypeCheck.genConstraintsExpr( + conf, cluster.getRexBuilder(), getTargetTable(qb, dest), updating(dest), inputRR); + if (constraintUDF == null) { + return null; + } + + RelNode constraintRel = genFilterRelNode(constraintUDF, srcRel, outerNameToPosMap, outerRR); + return new Pair<>(constraintRel, inputRR); + } + private AggregateCall convertGBAgg(AggregateInfo agg, List gbChildProjLst, HashMap rexNodeToPosMap, Integer childProjLstIndx) throws SemanticException { // 1. Get agg fn ret type in Calcite @@ -5127,6 +5162,13 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB, selectRel = selPair.getKey(); srcRel = (selectRel == null) ? srcRel : selectRel; + // Build Rel for Constraint checks + Pair constraintPair = + genConstraintFilterLogicalPlan(qb, srcRel, outerNameToPosMap, outerRR); + if (constraintPair != null) { + selPair = constraintPair; + } + // 6. Build Rel for OB Clause obRel = genOBLogicalPlan(qb, selPair, outerMostQB); srcRel = (obRel == null) ? srcRel : obRel; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0de3730351..2da3786d7c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -58,8 +58,6 @@ import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.util.ImmutableBitSet; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -162,14 +160,12 @@ import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; -import org.apache.hadoop.hive.ql.metadata.CheckConstraint; import org.apache.hadoop.hive.ql.metadata.DefaultConstraint; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; -import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Table; @@ -7113,109 +7109,6 @@ private void setStatsForNonNativeTable(String dbName, String tableName) throws S this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc))); } - - private void replaceColumnReference(ASTNode checkExpr, Map col2Col, - RowResolver inputRR){ - if(checkExpr.getType() == HiveParser.TOK_TABLE_OR_COL) { - ASTNode oldColChild = (ASTNode)(checkExpr.getChild(0)); - String oldColRef = oldColChild.getText().toLowerCase(); - assert(col2Col.containsKey(oldColRef)); - String internalColRef = col2Col.get(oldColRef); - String fullQualColRef[] = inputRR.reverseLookup(internalColRef); - String newColRef = fullQualColRef[1]; - checkExpr.deleteChild(0); - checkExpr.addChild(ASTBuilder.createAST(oldColChild.getType(), newColRef)); - } - else { - for(int i=0; i< checkExpr.getChildCount(); i++) { - replaceColumnReference((ASTNode)(checkExpr.getChild(i)), col2Col, inputRR); - } - } - } - - private ExprNodeDesc getCheckConstraintExpr(Table tbl, Operator input, RowResolver inputRR, String dest) - throws SemanticException{ - - CheckConstraint cc = null; - try { - cc = Hive.get().getEnabledCheckConstraints(tbl.getDbName(), tbl.getTableName()); - } - catch(HiveException e) { - throw new SemanticException(e); - } - if(cc == null || cc.getCheckConstraints().isEmpty()) { - return null; - } - - // build a map which tracks the name of column in input's signature to corresponding table column name - // this will be used to replace column references in CHECK expression AST with corresponding column name - // in input - Map col2Cols = new HashMap<>(); - List colInfos = input.getSchema().getSignature(); - int colIdx = 0; - if(updating(dest)) { - // if this is an update we need to skip the first col since it is row id - colIdx = 1; - } - for(FieldSchema fs: tbl.getCols()) { - // since SQL is case insenstive just to make sure that the comparison b/w column names - // and check expression's column reference work convert the key to lower case - col2Cols.put(fs.getName().toLowerCase(), colInfos.get(colIdx).getInternalName()); - colIdx++; - } - - List checkExprStrs = cc.getCheckExpressionList(); - TypeCheckCtx typeCheckCtx = new TypeCheckCtx(inputRR); - ExprNodeDesc checkAndExprs = null; - for(String checkExprStr:checkExprStrs) { - try { - ParseDriver parseDriver = new ParseDriver(); - ASTNode checkExprAST = parseDriver.parseExpression(checkExprStr); - //replace column references in checkExprAST with corresponding columns in input - replaceColumnReference(checkExprAST, col2Cols, inputRR); - Map genExprs = ExprNodeTypeCheck - .genExprNode(checkExprAST, typeCheckCtx); - ExprNodeDesc checkExpr = genExprs.get(checkExprAST); - // Check constraint fails only if it evaluates to false, NULL/UNKNOWN should evaluate to TRUE - ExprNodeDesc notFalseCheckExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(). - getFuncExprNodeDesc("isnotfalse", checkExpr); - if(checkAndExprs == null) { - checkAndExprs = notFalseCheckExpr; - } - else { - checkAndExprs = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor(). - getFuncExprNodeDesc("and", checkAndExprs, notFalseCheckExpr); - } - } catch(Exception e) { - throw new SemanticException(e); - } - } - return checkAndExprs; - } - - private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException{ - final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints( - tbl.getDbName(), tbl.getTableName()); - ImmutableBitSet bitSet = null; - if(nnc == null || nnc.getNotNullConstraints().isEmpty()) { - return bitSet; - } - // Build the bitset with not null columns - ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); - for (String nnCol : nnc.getNotNullConstraints().values()) { - int nnPos = -1; - for (int i = 0; i < tbl.getCols().size(); i++) { - if (tbl.getCols().get(i).getName().equals(nnCol)) { - nnPos = i; - builder.set(nnPos); - break; - } - } - } - bitSet = builder.build(); - return bitSet; - } - private boolean mergeCardinalityViolationBranch(final Operator input) { if(input instanceof SelectOperator) { SelectOperator selectOp = (SelectOperator)input; @@ -7231,104 +7124,43 @@ private boolean mergeCardinalityViolationBranch(final Operator input) { } private Operator genConstraintsPlan(String dest, QB qb, Operator input) throws SemanticException { - if(deleting(dest)) { + if (deleting(dest)) { // for DELETE statements NOT NULL constraint need not be checked return input; } //MERGE statements could have inserted a cardinality violation branch, we need to avoid that - if(mergeCardinalityViolationBranch(input)){ + if (mergeCardinalityViolationBranch(input)) { return input; } // if this is an insert into statement we might need to add constraint check - Table targetTable = null; - Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); - if(dest_type == QBMetaData.DEST_TABLE) { - targetTable= qb.getMetaData().getDestTableForAlias(dest); - - } - else if(dest_type == QBMetaData.DEST_PARTITION){ - Partition dest_part = qb.getMetaData().getDestPartitionForAlias(dest); - targetTable = dest_part.getTable(); - - } - else { - throw new SemanticException("Generating constraint check plan: Invalid target type: " + dest); - } - + assert (input.getParentOperators().size() == 1); RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - ExprNodeDesc nullConstraintExpr = getNotNullConstraintExpr(targetTable, input, dest); - ExprNodeDesc checkConstraintExpr = getCheckConstraintExpr(targetTable, input, inputRR, dest); - - ExprNodeDesc combinedConstraintExpr = null; - if(nullConstraintExpr != null && checkConstraintExpr != null) { - assert (input.getParentOperators().size() == 1); - combinedConstraintExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() - .getFuncExprNodeDesc("and", nullConstraintExpr, checkConstraintExpr); - - } - else if(nullConstraintExpr != null) { - combinedConstraintExpr = nullConstraintExpr; - } - else if(checkConstraintExpr != null) { - combinedConstraintExpr = checkConstraintExpr; - } + Table targetTable = getTargetTable(qb, dest); + ExprNodeDesc combinedConstraintExpr = + ExprNodeTypeCheck.genConstraintsExpr(conf, targetTable, updating(dest), inputRR); if (combinedConstraintExpr != null) { - ExprNodeDesc constraintUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() - .getFuncExprNodeDesc("enforce_constraint", combinedConstraintExpr); return putOpInsertMap(OperatorFactory.getAndMakeChild( - new FilterDesc(constraintUDF, false), new RowSchema( + new FilterDesc(combinedConstraintExpr, false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); } return input; } - private ExprNodeDesc getNotNullConstraintExpr(Table targetTable, Operator input, String dest) throws SemanticException { - boolean forceNotNullConstraint = conf.getBoolVar(ConfVars.HIVE_ENFORCE_NOT_NULL_CONSTRAINT); - if(!forceNotNullConstraint) { - return null; - } - - ImmutableBitSet nullConstraintBitSet = null; - try { - nullConstraintBitSet = getEnabledNotNullConstraints(targetTable); - } catch (Exception e) { - if (e instanceof SemanticException) { - throw (SemanticException) e; - } else { - throw (new RuntimeException(e)); - } - } + protected Table getTargetTable(QB qb, String dest) throws SemanticException { + Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); + if (dest_type == QBMetaData.DEST_TABLE) { + return qb.getMetaData().getDestTableForAlias(dest); - if(nullConstraintBitSet == null) { - return null; - } - List colInfos = input.getSchema().getSignature(); + } else if (dest_type == QBMetaData.DEST_PARTITION) { + Partition dest_part = qb.getMetaData().getDestPartitionForAlias(dest); + return dest_part.getTable(); - ExprNodeDesc currUDF = null; - // Add NOT NULL constraints - int constraintIdx = 0; - for(int colExprIdx=0; colExprIdx < colInfos.size(); colExprIdx++) { - if(updating(dest) && colExprIdx == 0) { - // for updates first column is _rowid - continue; - } - if (nullConstraintBitSet.indexOf(constraintIdx) != -1) { - ExprNodeDesc currExpr = ExprNodeTypeCheck.toExprNode(colInfos.get(colExprIdx), null); - ExprNodeDesc isNotNullUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() - .getFuncExprNodeDesc("isnotnull", currExpr); - if (currUDF != null) { - currUDF = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() - .getFuncExprNodeDesc("and", currUDF, isNotNullUDF); - } else { - currUDF = isNotNullUDF; - } - } - constraintIdx++; + } else { + throw new SemanticException("Generating constraint check plan: Invalid target type: " + dest); } - return currUDF; } private Path getDestinationFilePath(final String destinationFile, boolean isMmTable) { @@ -15054,7 +14886,7 @@ private boolean isAcidOutputFormat(Class of) { } } - private boolean updating(String destination) { + protected boolean updating(String destination) { return destination.startsWith(Context.DestClausePrefix.UPDATE.toString()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/type/ConstraintExprGenerator.java ql/src/java/org/apache/hadoop/hive/ql/parse/type/ConstraintExprGenerator.java new file mode 100644 index 0000000000..47d1bd5d22 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/type/ConstraintExprGenerator.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.type; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.metadata.CheckConstraint; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +public class ConstraintExprGenerator { + private final HiveConf conf; + private final TypeCheckProcFactory typeCheckProcFactory; + private final TypeCheckProcFactory.DefaultExprProcessor exprProcessor; + + public ConstraintExprGenerator(HiveConf conf, TypeCheckProcFactory typeCheckProcFactory) { + this.conf = conf; + this.typeCheckProcFactory = typeCheckProcFactory; + this.exprProcessor = typeCheckProcFactory.getDefaultExprProcessor(); + } + + public T genConstraintsExpr( + Table targetTable, boolean updateStatement, RowResolver inputRR) + throws SemanticException { + List inputColumnInfoList = inputRR.getColumnInfos(); + T nullConstraintExpr = getNotNullConstraintExpr(targetTable, inputRR, updateStatement); + T checkConstraintExpr = getCheckConstraintExpr(targetTable, inputColumnInfoList, inputRR, updateStatement); + T combinedConstraintExpr = null; + if (nullConstraintExpr != null && checkConstraintExpr != null) { + combinedConstraintExpr = exprProcessor.getFuncExprNodeDesc("and", nullConstraintExpr, checkConstraintExpr); + } else if (nullConstraintExpr != null) { + combinedConstraintExpr = nullConstraintExpr; + } else if (checkConstraintExpr != null) { + combinedConstraintExpr = checkConstraintExpr; + } + + if (combinedConstraintExpr == null) { + return null; + } + return exprProcessor.getFuncExprNodeDesc("enforce_constraint", combinedConstraintExpr); + } + + private T getNotNullConstraintExpr( + Table targetTable, RowResolver inputRR, boolean isUpdateStatement) + throws SemanticException { + boolean forceNotNullConstraint = conf.getBoolVar(HiveConf.ConfVars.HIVE_ENFORCE_NOT_NULL_CONSTRAINT); + if(!forceNotNullConstraint) { + return null; + } + + ImmutableBitSet nullConstraintBitSet; + try { + nullConstraintBitSet = getEnabledNotNullConstraints(targetTable); + } catch (SemanticException e) { + throw e; + } catch (Exception e) { + throw (new RuntimeException(e)); + } + + if(nullConstraintBitSet == null) { + return null; + } + + T currUDF = null; + int constraintIdx = 0; + List inputColInfos = inputRR.getColumnInfos(); + for(int colExprIdx = 0; colExprIdx < inputColInfos.size(); colExprIdx++) { + if (isUpdateStatement && colExprIdx == 0) { + // for updates first column is _rowid + continue; + } + if (nullConstraintBitSet.indexOf(constraintIdx) != -1) { + T currExpr = typeCheckProcFactory.exprFactory.createColumnRefExpr(inputColInfos.get(colExprIdx), inputRR, 0); + T isNotNullUDF = exprProcessor.getFuncExprNodeDesc("isnotnull", currExpr); + if (currUDF != null) { + currUDF = exprProcessor.getFuncExprNodeDesc("and", currUDF, isNotNullUDF); + } else { + currUDF = isNotNullUDF; + } + } + constraintIdx++; + } + return currUDF; + } + + private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException { + final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints( + tbl.getDbName(), tbl.getTableName()); + if(nnc == null || nnc.getNotNullConstraints().isEmpty()) { + return null; + } + // Build the bitset with not null columns + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + for (String nnCol : nnc.getNotNullConstraints().values()) { + int nnPos = -1; + for (int i = 0; i < tbl.getCols().size(); i++) { + if (tbl.getCols().get(i).getName().equals(nnCol)) { + nnPos = i; + builder.set(nnPos); + break; + } + } + } + return builder.build(); + } + + private T getCheckConstraintExpr( + Table tbl, List inputColInfos, RowResolver inputRR, boolean isUpdateStatement) + throws SemanticException{ + + CheckConstraint cc = null; + try { + cc = Hive.get().getEnabledCheckConstraints(tbl.getDbName(), tbl.getTableName()); + } + catch(HiveException e) { + throw new SemanticException(e); + } + if(cc == null || cc.getCheckConstraints().isEmpty()) { + return null; + } + + // build a map which tracks the name of column in input's signature to corresponding table column name + // this will be used to replace column references in CHECK expression AST with corresponding column name + // in input + Map col2Cols = new HashMap<>(); + int colIdx = 0; + if(isUpdateStatement) { + // if this is an update we need to skip the first col since it is row id + colIdx = 1; + } + for(FieldSchema fs: tbl.getCols()) { + // since SQL is case insenstive just to make sure that the comparison b/w column names + // and check expression's column reference work convert the key to lower case + col2Cols.put(fs.getName().toLowerCase(), inputColInfos.get(colIdx).getInternalName()); + colIdx++; + } + + List checkExprStrs = cc.getCheckExpressionList(); + TypeCheckCtx typeCheckCtx = new TypeCheckCtx(inputRR); + T checkAndExprs = null; + for(String checkExprStr:checkExprStrs) { + try { + ParseDriver parseDriver = new ParseDriver(); + ASTNode checkExprAST = parseDriver.parseExpression(checkExprStr); + //replace column references in checkExprAST with corresponding columns in input + replaceColumnReference(checkExprAST, col2Cols, inputRR); + Map genExprs = typeCheckProcFactory.genExprNode(checkExprAST, typeCheckCtx); + T checkExpr = genExprs.get(checkExprAST); + // Check constraint fails only if it evaluates to false, NULL/UNKNOWN should evaluate to TRUE + T notFalseCheckExpr = exprProcessor.getFuncExprNodeDesc("isnotfalse", checkExpr); + if(checkAndExprs == null) { + checkAndExprs = notFalseCheckExpr; + } + else { + checkAndExprs = exprProcessor.getFuncExprNodeDesc("and", checkAndExprs, notFalseCheckExpr); + } + } catch(Exception e) { + throw new SemanticException(e); + } + } + return checkAndExprs; + } + + private void replaceColumnReference(ASTNode checkExpr, Map col2Col, + RowResolver inputRR){ + if(checkExpr.getType() == HiveParser.TOK_TABLE_OR_COL) { + ASTNode oldColChild = (ASTNode)(checkExpr.getChild(0)); + String oldColRef = oldColChild.getText().toLowerCase(); + assert(col2Col.containsKey(oldColRef)); + String internalColRef = col2Col.get(oldColRef); + String[] fullQualColRef = inputRR.reverseLookup(internalColRef); + String newColRef = fullQualColRef[1]; + checkExpr.deleteChild(0); + checkExpr.addChild(ASTBuilder.createAST(oldColChild.getType(), newColRef)); + } + else { + for(int i=0; i< checkExpr.getChildCount(); i++) { + replaceColumnReference((ASTNode)(checkExpr.getChild(i)), col2Col, inputRR); + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java index 3e3d331412..1ecdfdc0db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprNodeTypeCheck.java @@ -19,7 +19,10 @@ package org.apache.hadoop.hive.ql.parse.type; import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -75,4 +78,10 @@ public static ExprNodeDesc toExprNode(ColumnInfo columnInfo, RowResolver rowReso return factory.toExpr(columnInfo, rowResolver, 0); } + public static ExprNodeDesc genConstraintsExpr( + HiveConf conf, Table targetTable, boolean updateStatement, RowResolver inputRR) + throws SemanticException { + return new ConstraintExprGenerator<>(conf, new TypeCheckProcFactory<>(new ExprNodeDescExprFactory())) + .genConstraintsExpr(targetTable, updateStatement, inputRR); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeTypeCheck.java ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeTypeCheck.java index c9131ec018..745c205827 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeTypeCheck.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeTypeCheck.java @@ -20,7 +20,9 @@ import java.util.Map; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -67,4 +69,10 @@ public static RexNode toExprNode(ColumnInfo columnInfo, RowResolver rowResolver, return factory.toExpr(columnInfo, rowResolver, offset); } + public static RexNode genConstraintsExpr( + HiveConf conf, RexBuilder rexBuilder, Table targetTable, boolean updateStatement, RowResolver inputRR) + throws SemanticException { + return new ConstraintExprGenerator<>(conf, new TypeCheckProcFactory<>(new RexNodeExprFactory(rexBuilder))) + .genConstraintsExpr(targetTable, updateStatement, inputRR); + } } diff --git ql/src/test/results/clientnegative/update_notnull_constraint.q.out ql/src/test/results/clientnegative/update_notnull_constraint.q.out index 32905378e7..86bfc67480 100644 --- ql/src/test/results/clientnegative/update_notnull_constraint.q.out +++ ql/src/test/results/clientnegative/update_notnull_constraint.q.out @@ -21,9 +21,4 @@ POSTHOOK: Output: default@acid_uami POSTHOOK: Lineage: acid_uami.de SCRIPT [] POSTHOOK: Lineage: acid_uami.i SCRIPT [] POSTHOOK: Lineage: acid_uami.vc SCRIPT [] -PREHOOK: query: UPDATE acid_uami set de=null where i=1 -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_uami -PREHOOK: Output: default@acid_uami -#### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: DataConstraintViolationError org.apache.hadoop.hive.ql.exec.errors.DataConstraintViolationError: Either CHECK or NOT NULL constraint violated! diff --git ql/src/test/results/clientpositive/llap/check_constraint.q.out ql/src/test/results/clientpositive/llap/check_constraint.q.out index bc5d361859..e4fe16427f 100644 --- ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -2070,10 +2070,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_uami_n0 - filterExpr: (de) IN (103, 119) (type: boolean) + filterExpr: ((de) IN (103, 119) and enforce_constraint((893.14 >= CAST( i AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (de) IN (103, 119) (type: boolean) + predicate: ((de) IN (103, 119) and enforce_constraint((893.14 >= CAST( i AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct), i (type: int), vc (type: varchar(128)) @@ -2095,18 +2095,15 @@ STAGE PLANS: expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 893.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: enforce_constraint((_col2 is not null and (_col2 >= CAST( _col1 AS decimal(5,2))) is not false)) (type: boolean) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.acid_uami_n0 - Write Type: UPDATE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami_n0 + Write Type: UPDATE Stage: Stage-2 Dependency Collection @@ -2202,25 +2199,18 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 893.14 (type: decimal(5,2)), 'apache_hive' (type: string) + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 893.14 (type: decimal(5,2)), 'apache_hive' (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: enforce_constraint(_col2 is not null) (type: boolean) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: struct), _col1 (type: int), _col2 (type: decimal(5,2)), CAST( _col3 AS varchar(128)) (type: varchar(128)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.acid_uami_n0 - Write Type: UPDATE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami_n0 + Write Type: UPDATE Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/default_constraint.q.out ql/src/test/results/clientpositive/llap/default_constraint.q.out index a04da4e2b4..858dd5ab63 100644 --- ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -1902,7 +1902,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: enforce_constraint(127Y is not null) (type: boolean) + predicate: enforce_constraint(_col0 is not null) (type: boolean) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), CAST( _col5 AS decimal(9,2)) (type: decimal(9,2)) @@ -2248,7 +2248,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: enforce_constraint(108Y is not null) (type: boolean) + predicate: enforce_constraint(_col0 is not null) (type: boolean) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), CAST( _col5 AS decimal(9,2)) (type: decimal(9,2)) diff --git ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out index e96363fe29..6abd6f3c82 100644 --- ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out +++ ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -3393,21 +3393,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_uami_n1 - filterExpr: (de) IN (109.23, 119.23) (type: boolean) + filterExpr: ((de) IN (109.23, 119.23) and enforce_constraint(vc is not null)) (type: boolean) Statistics: Num rows: 1002 Data size: 225450 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (de) IN (109.23, 119.23) (type: boolean) - Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((de) IN (109.23, 119.23) and enforce_constraint(vc is not null)) (type: boolean) + Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ROW__ID (type: struct), i (type: int), vc (type: varchar(128)) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col3 (type: varchar(128)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -3415,25 +3415,18 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(3,2)), VALUE._col1 (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: enforce_constraint((_col2 is not null and _col3 is not null)) (type: boolean) + Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: int), CAST( _col2 AS decimal(5,2)) (type: decimal(5,2)), _col3 (type: varchar(128)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.acid_uami_n1 - Write Type: UPDATE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami_n1 + Write Type: UPDATE Stage: Stage-2 Dependency Collection @@ -3499,21 +3492,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_uami_n1 - filterExpr: (de = 3.14) (type: boolean) + filterExpr: ((de = 3.14) and enforce_constraint((i is not null and vc is not null))) (type: boolean) Statistics: Num rows: 1002 Data size: 225450 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (de = 3.14) (type: boolean) - Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((de = 3.14) and enforce_constraint((i is not null and vc is not null))) (type: boolean) + Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ROW__ID (type: struct), i (type: int), vc (type: varchar(128)) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col3 (type: varchar(128)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -3521,25 +3514,18 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14159 (type: decimal(6,5)), VALUE._col1 (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: enforce_constraint((_col1 is not null and _col3 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: int), CAST( _col2 AS decimal(5,2)) (type: decimal(5,2)), _col3 (type: varchar(128)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.acid_uami_n1 - Write Type: UPDATE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami_n1 + Write Type: UPDATE Stage: Stage-2 Dependency Collection