diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b7d3e99e1a..578631cadd 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1570,6 +1570,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "not a multiple of each other, bucketed map-side join cannot be performed, and the\n" + "query will fail if hive.enforce.bucketmapjoin is set to true."), + HIVE_ENFORCE_NOT_NULL_CONSTRAINT("hive.constraint.notnull.enforce", true, + "Should \"IS NOT NULL \" constraint be enforced?"), + HIVE_AUTO_SORTMERGE_JOIN("hive.auto.convert.sortmerge.join", false, "Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join."), HIVE_AUTO_SORTMERGE_JOIN_REDUCE("hive.auto.convert.sortmerge.join.reduce.side", true, diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index d86ff58840..ba2a08d219 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -538,6 +538,7 @@ minillaplocal.query.files=\ dynpart_sort_opt_vectorization.q,\ dynpart_sort_optimization.q,\ dynpart_sort_optimization_acid.q,\ + enforce_constraint_notnull.q,\ escape1.q,\ escape2.q,\ exchgpartition2lel.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 76e85636d1..f7801bb5a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -214,6 +214,7 @@ system.registerUDF("rand", UDFRand.class, false); system.registerGenericUDF("abs", GenericUDFAbs.class); system.registerGenericUDF("sq_count_check", GenericUDFSQCountCheck.class); + system.registerGenericUDF("enforce_constraint", GenericUDFEnforceNotNullConstraint.class); system.registerGenericUDF("pmod", GenericUDFPosMod.class); system.registerUDF("ln", UDFLn.class, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 63bcedc000..afd87c0fa5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -4774,6 +4774,30 @@ public NotNullConstraint getReliableNotNullConstraints(String dbName, String tbl return getNotNullConstraints(dbName, tblName, true); } + /** + * Get not null constraints associated with the table that are enabled/enforced + * + * @param dbName Database Name + * @param tblName Table Name + * @return Not null constraints associated with the table. + * @throws HiveException + */ + public NotNullConstraint getEnabledNotNullConstraints(String dbName, String tblName) + throws HiveException { + try { + List notNullConstraints = getMSC().getNotNullConstraints( + new NotNullConstraintsRequest(dbName, tblName)); + if (notNullConstraints != null && !notNullConstraints.isEmpty()) { + notNullConstraints = notNullConstraints.stream() + .filter(nnc -> nnc.isEnable_cstr()) + .collect(Collectors.toList()); + } + return new NotNullConstraint(notNullConstraints, tblName, dbName); + } catch (Exception e) { + throw new HiveException(e); + } + } + private NotNullConstraint getNotNullConstraints(String dbName, String tblName, boolean onlyReliable) throws HiveException { try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 372cfad866..2206dfb9e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -778,8 +778,8 @@ private static void generateConstraintInfos(ASTNode child, List columnNa constraintName = unescapeIdentifier(grandChild.getChild(0).getText().toLowerCase()); } else if (type == HiveParser.TOK_ENABLE) { enable = true; - // validate is true by default if we enable the constraint - validate = true; + // validate is false by default if we enable the constraint + validate = false; } else if (type == HiveParser.TOK_DISABLE) { enable = false; // validate is false by default if we disable the constraint @@ -792,11 +792,7 @@ private static void generateConstraintInfos(ASTNode child, List columnNa rely = true; } } - if (enable) { - throw new SemanticException( - ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("ENABLE/ENFORCED feature not supported yet. " - + "Please use DISABLE/NOT ENFORCED instead.")); - } + if (validate) { throw new SemanticException( ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("VALIDATE feature not supported yet. " diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dbf9363d11..06712334d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -52,6 +52,7 @@ import org.antlr.runtime.tree.TreeWizard; import org.antlr.runtime.tree.TreeWizard.ContextVisitor; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -127,6 +128,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; +import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Table; @@ -136,6 +138,7 @@ import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPostProc; import org.apache.hadoop.hive.ql.optimizer.lineage.Generator; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; @@ -202,6 +205,7 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; @@ -210,6 +214,7 @@ import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCardinalityViolation; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; @@ -6620,6 +6625,128 @@ private void setStatsForNonNativeTable(Table tab) throws SemanticException { this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); } + private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException{ + List nullConstraints = new ArrayList<>(); + final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints( + tbl.getDbName(), tbl.getTableName()); + ImmutableBitSet bitSet = null; + if(nnc == null || nnc.getNotNullConstraints().isEmpty()) { + return bitSet; + } + // Build the bitset with not null columns + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + for (String nnCol : nnc.getNotNullConstraints().values()) { + int nnPos = -1; + for (int i = 0; i < tbl.getCols().size(); i++) { + if (tbl.getCols().get(i).getName().equals(nnCol)) { + nnPos = i; + builder.set(nnPos); + break; + } + } + } + bitSet = builder.build(); + return bitSet; + } + + private boolean mergeCardinalityViolationBranch(final Operator input) { + if(input instanceof SelectOperator) { + SelectOperator selectOp = (SelectOperator)input; + if(selectOp.getConf().getColList().size() == 1) { + ExprNodeDesc colExpr = selectOp.getConf().getColList().get(0); + if(colExpr instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc)colExpr ; + if(func.getGenericUDF() instanceof GenericUDFCardinalityViolation) { + return true; + } + } + } + } + return false; + } + + private Operator + genIsNotNullConstraint(String dest, QB qb, Operator input) + throws SemanticException { + + boolean forceNotNullConstraint = conf.getBoolVar(ConfVars.HIVE_ENFORCE_NOT_NULL_CONSTRAINT); + if(!forceNotNullConstraint) { + return input; + } + + if(deleting(dest)) { + // for DELETE statements NOT NULL constraint need not be checked + return input; + } + + //MERGE statements could have inserted a cardinality violation branch, we need to avoid that + if(mergeCardinalityViolationBranch(input)){ + return input; + } + + // if this is an insert into statement we might need to add constraint check + Table targetTable = null; + Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); + if(dest_type == QBMetaData.DEST_TABLE) { + targetTable= qb.getMetaData().getDestTableForAlias(dest); + + } + else if(dest_type == QBMetaData.DEST_PARTITION){ + Partition dest_part = qb.getMetaData().getDestPartitionForAlias(dest); + targetTable = dest_part.getTable(); + + } + else { + throw new SemanticException("Generating NOT NULL constraint check: Invalid target type"); + } + ImmutableBitSet nullConstraintBitSet = null; + try { + nullConstraintBitSet = getEnabledNotNullConstraints(targetTable); + } catch (Exception e) { + if (e instanceof SemanticException) { + throw (SemanticException) e; + } else { + throw (new RuntimeException(e)); + } + } + if(nullConstraintBitSet == null) { + return input; + } + List colInfos = input.getSchema().getSignature(); + + ExprNodeDesc currUDF = null; + int constraintIdx = 0; + for(int colExprIdx=0; colExprIdx < colInfos.size(); colExprIdx++) { + if(updating(dest) && colExprIdx == 0) { + // for updates first column is _rowid + continue; + } + if (nullConstraintBitSet.indexOf(constraintIdx) != -1) { + ExprNodeDesc currExpr = TypeCheckProcFactory.toExprNodeDesc(colInfos.get(colExprIdx)); + ExprNodeDesc isNotNullUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("isnotnull", currExpr); + ExprNodeDesc constraintUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("enforce_constraint", isNotNullUDF); + if (currUDF != null) { + currUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("and", currUDF, constraintUDF); + } else { + currUDF = constraintUDF; + } + } + constraintIdx++; + } + if (currUDF != null) { + assert (input.getParentOperators().size() == 1); + RowResolver inputRR = opParseCtx.get(input).getRowResolver(); + Operator newConstraintFilter = putOpInsertMap(OperatorFactory.getAndMakeChild( + new FilterDesc(currUDF, false), new RowSchema( + inputRR.getColumnInfos()), input), inputRR); + + return newConstraintFilter; + } + return input; + } @SuppressWarnings("nls") protected Operator genFileSinkPlan(String dest, QB qb, Operator input) throws SemanticException { @@ -6705,6 +6832,9 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // this table_desc does not contain the partitioning columns table_desc = Utilities.getTableDesc(dest_tab); + // Add NOT NULL constraint check + input = genIsNotNullConstraint(dest, qb, input); + // Add sorting/bucketing if needed input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx); @@ -6784,6 +6914,9 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } table_desc = Utilities.getTableDesc(dest_tab); + // Add NOT NULL constraint check + input = genIsNotNullConstraint(dest, qb, input); + // Add sorting/bucketing if needed input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 9fcde76d21..14217e3978 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -701,7 +701,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } - private static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) { + static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) { ObjectInspector inspector = colInfo.getObjectInspector(); if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index e3a9e62bdb..2accad3a7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -139,8 +139,10 @@ private static void removeAllCandidates(OpWalkerInfo owi) { } private static void removeOperator(Operator operator) { - List> children = operator.getChildOperators(); - List> parents = operator.getParentOperators(); + // since removeParent/removeChild updates the childOperators and parentOperators list in place + // we need to make a copy of list to iterator over them + List> children = new ArrayList<>(operator.getChildOperators()); + List> parents = new ArrayList<>(operator.getParentOperators()); for (Operator parent : parents) { parent.getChildOperators().addAll(children); parent.removeChild(operator); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java new file mode 100644 index 0000000000..5b676ffe94 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsDecimalToDecimal; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsDoubleToDouble; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsLongToLong; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.LongWritable; + +/** + * GenericUDFAbs. + * + */ +@Description(name = "enforce_constraint", + value = "_FUNC_(x) - Internal UDF to enforce NOT NULL constraint", + extended = "For internal use only") +@VectorizedExpressions({FuncAbsLongToLong.class, FuncAbsDoubleToDouble.class, FuncAbsDecimalToDecimal.class}) +public class GenericUDFEnforceNotNullConstraint extends GenericUDF { + private final BooleanWritable resultBool = new BooleanWritable(); + private transient BooleanObjectInspector boi; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length > 1) { + throw new UDFArgumentLengthException( + "Invalid number of arguments. enforce_constraint UDF expected one argument but received: " + arguments.length); + } + + boi = (BooleanObjectInspector) arguments[0]; + return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + + Object a = arguments[0].get(); + boolean result = boi.get(a); + + if(result == false) { + throw new UDFArgumentLengthException( + "NOT NULL constraint violated!"); + } + resultBool.set(true); + return resultBool; + } + + @Override + protected String getFuncName() { + return "enforce_constraint"; + } + + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); + } + +} diff --git a/ql/src/test/queries/clientnegative/create_with_constraints_enable.q b/ql/src/test/queries/clientnegative/create_with_constraints_enable.q deleted file mode 100644 index 59ebb1e9b2..0000000000 --- a/ql/src/test/queries/clientnegative/create_with_constraints_enable.q +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE table1 (a STRING, b STRING, primary key (a) enable); diff --git a/ql/src/test/queries/clientnegative/create_with_constraints_enforced.q b/ql/src/test/queries/clientnegative/create_with_constraints_enforced.q deleted file mode 100644 index 5b90c0efc9..0000000000 --- a/ql/src/test/queries/clientnegative/create_with_constraints_enforced.q +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE table1 (a STRING, b STRING, primary key (a) enforced); diff --git a/ql/src/test/queries/clientnegative/insert_into_acid_notnull.q b/ql/src/test/queries/clientnegative/insert_into_acid_notnull.q new file mode 100644 index 0000000000..da9aba3239 --- /dev/null +++ b/ql/src/test/queries/clientnegative/insert_into_acid_notnull.q @@ -0,0 +1,9 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table acid_uami(i int, + de decimal(5,2) not null enforced, + vc varchar(128) not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + + +insert into table acid_uami select 1, null, null; diff --git a/ql/src/test/queries/clientnegative/insert_into_notnull_constraint.q b/ql/src/test/queries/clientnegative/insert_into_notnull_constraint.q new file mode 100644 index 0000000000..da35b02cd3 --- /dev/null +++ b/ql/src/test/queries/clientnegative/insert_into_notnull_constraint.q @@ -0,0 +1,3 @@ +create table nullConstraintCheck(i int NOT NULL enforced, j int); +insert into nullConstraintCheck values(null,2); + diff --git a/ql/src/test/queries/clientnegative/insert_multi_into_notnull.q b/ql/src/test/queries/clientnegative/insert_multi_into_notnull.q new file mode 100644 index 0000000000..2c4ec1eff2 --- /dev/null +++ b/ql/src/test/queries/clientnegative/insert_multi_into_notnull.q @@ -0,0 +1,6 @@ +create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING); +create table src_multi2 (i STRING, j STRING NOT NULL ENFORCED); + +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select key, null where key > 10 and key < 20; diff --git a/ql/src/test/queries/clientnegative/insert_overwrite_notnull_constraint.q b/ql/src/test/queries/clientnegative/insert_overwrite_notnull_constraint.q new file mode 100644 index 0000000000..64fe4e4ca5 --- /dev/null +++ b/ql/src/test/queries/clientnegative/insert_overwrite_notnull_constraint.q @@ -0,0 +1,3 @@ +create table nullConstraintCheck(i int NOT NULL enforced, j int); +insert overwrite table nullConstraintCheck values(null,2); + diff --git a/ql/src/test/queries/clientpositive/enforce_constraint_notnull.q b/ql/src/test/queries/clientpositive/enforce_constraint_notnull.q new file mode 100644 index 0000000000..83e061ce03 --- /dev/null +++ b/ql/src/test/queries/clientpositive/enforce_constraint_notnull.q @@ -0,0 +1,180 @@ +-- SIMPLE TABLE +-- create table with first and last column with not null +CREATE TABLE table1 (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED); + +-- insert value tuples +explain INSERT INTO table1 values('not', 'null', 'constraint'); +INSERT INTO table1 values('not', 'null', 'constraint'); +SELECT * FROM table1; + +-- insert from select +explain INSERT INTO table1 select key, src.value, value from src; +INSERT INTO table1 select key, src.value, value from src; +SELECT * FROM table1; + +-- insert overwrite +explain INSERT OVERWRITE TABLE table1 select src.*, value from src; +INSERT OVERWRITE TABLE table1 select src.*, value from src; +SELECT * FROM table1; + +-- insert overwrite with if not exists +explain INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src; +INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src; +SELECT * FROM table1; + +DROP TABLE table1; + +-- multi insert +create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING); +create table src_multi2 (i STRING, j STRING NOT NULL ENABLE); + +explain +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20; + + +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20; + +explain +from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20; + +from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20; + +-- ACID TABLE +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +-- SORT_QUERY_RESULTS +create table acid_uami(i int, + de decimal(5,2) constraint nn1 not null enforced, + vc varchar(128) constraint nn2 not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + +-- insert into values +explain insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow'); +insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow'); +select * from acid_uami; + + --insert into select +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src; +insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src; + +-- select with limit +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src limit 2; + +-- select with order by +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + order by key limit 2; + +-- select with group by +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + group by key, value order by key limit 2; + + --overwrite +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src; +insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src; + +-- update +explain update acid_uami set de = 3.14 where de = 109.23 or de = 119.23; +update acid_uami set de = 3.14 where de = 109.23 or de = 119.23; + +ALTER table acid_uami drop constraint nn1; +ALTER table acid_uami CHANGE i i int constraint nn0 not null enforced; + +explain update acid_uami set de = 3.14159 where de = 3.14 ; +update acid_uami set de = 3.14159 where de = 3.14 ; + +-- multi insert +explain +from src +insert overwrite table acid_uami select cast(key as int), cast(key as decimal(5,2)), value where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20; + +set hive.exec.dynamic.partition.mode=nonstrict; +-- Table with partition +CREATE TABLE tablePartitioned (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED) + PARTITIONED BY (p1 STRING, p2 INT NOT NULL ENABLE); + +-- Insert into +explain INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint'); +INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint'); + +-- Insert as select +explain INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10; +INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10; + +select * from tablePartitioned; + +-- multi insert +explain +from src +INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, 'yesterday' as p1, 3 as p2 +insert overwrite table src_multi2 select * where key > 10 and key < 20; + +DROP TABLE src_multi1; +DROP TABLE src_multi2; +DROP TABLE acid_uami; + +-- MERGE statements +set hive.mapred.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table nonacid (key int, a1 string, value string) stored as orc; + +create table masking_test (key int NOT NULL enable, a1 string, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true"); + +-- with cardinality check off +set hive.merge.cardinality.check=false; +explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value); + +-- with cardinality check on +set hive.merge.cardinality.check=true; +explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value); + +explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value); + +explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value); + +-- shouldn't have constraint enforcement +explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE; + +DROP TABLE masking_test; +DROP TABLE nonacid; + +-- Test drop constraint +create table table2(i int constraint nn5 not null enforced, j int); +explain insert into table2 values(2, 3); +alter table table2 drop constraint nn5; +explain insert into table2 values(2, 3); +DROP TABLE table2; + +-- temporary table +create temporary table tttemp(i int not null enforced); +explain insert into tttemp values(1); +explain insert into tttemp select cast(key as int) from src; +drop table tttemp; + diff --git a/ql/src/test/results/clientnegative/create_with_constraints_enable.q.out b/ql/src/test/results/clientnegative/create_with_constraints_enable.q.out deleted file mode 100644 index 2b244120e9..0000000000 --- a/ql/src/test/results/clientnegative/create_with_constraints_enable.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10326]: Invalid Constraint syntax ENABLE/ENFORCED feature not supported yet. Please use DISABLE/NOT ENFORCED instead. diff --git a/ql/src/test/results/clientnegative/create_with_constraints_enforced.q.out b/ql/src/test/results/clientnegative/create_with_constraints_enforced.q.out deleted file mode 100644 index 2b244120e9..0000000000 --- a/ql/src/test/results/clientnegative/create_with_constraints_enforced.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10326]: Invalid Constraint syntax ENABLE/ENFORCED feature not supported yet. Please use DISABLE/NOT ENFORCED instead. diff --git a/ql/src/test/results/clientnegative/insert_into_acid_notnull.q.out b/ql/src/test/results/clientnegative/insert_into_acid_notnull.q.out new file mode 100644 index 0000000000..721c7e7bfd --- /dev/null +++ b/ql/src/test/results/clientnegative/insert_into_acid_notnull.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create table acid_uami(i int, + de decimal(5,2) not null enforced, + vc varchar(128) not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uami +POSTHOOK: query: create table acid_uami(i int, + de decimal(5,2) not null enforced, + vc varchar(128) not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uami +FAILED: UDFArgumentException org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException: NOT NULL constraint violated! diff --git a/ql/src/test/results/clientnegative/insert_into_notnull_constraint.q.out b/ql/src/test/results/clientnegative/insert_into_notnull_constraint.q.out new file mode 100644 index 0000000000..6718ae0d1d --- /dev/null +++ b/ql/src/test/results/clientnegative/insert_into_notnull_constraint.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table nullConstraintCheck(i int NOT NULL enforced, j int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nullConstraintCheck +POSTHOOK: query: create table nullConstraintCheck(i int NOT NULL enforced, j int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nullConstraintCheck +FAILED: UDFArgumentException org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException: NOT NULL constraint violated! diff --git a/ql/src/test/results/clientnegative/insert_multi_into_notnull.q.out b/ql/src/test/results/clientnegative/insert_multi_into_notnull.q.out new file mode 100644 index 0000000000..d54e6cf3c1 --- /dev/null +++ b/ql/src/test/results/clientnegative/insert_multi_into_notnull.q.out @@ -0,0 +1,17 @@ +PREHOOK: query: create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: create table src_multi2 (i STRING, j STRING NOT NULL ENFORCED) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 (i STRING, j STRING NOT NULL ENFORCED) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +FAILED: UDFArgumentException org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException: NOT NULL constraint violated! diff --git a/ql/src/test/results/clientnegative/insert_overwrite_notnull_constraint.q.out b/ql/src/test/results/clientnegative/insert_overwrite_notnull_constraint.q.out new file mode 100644 index 0000000000..6718ae0d1d --- /dev/null +++ b/ql/src/test/results/clientnegative/insert_overwrite_notnull_constraint.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table nullConstraintCheck(i int NOT NULL enforced, j int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nullConstraintCheck +POSTHOOK: query: create table nullConstraintCheck(i int NOT NULL enforced, j int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nullConstraintCheck +FAILED: UDFArgumentException org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException: NOT NULL constraint violated! diff --git a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out new file mode 100644 index 0000000000..52a0089a2a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -0,0 +1,5729 @@ +PREHOOK: query: CREATE TABLE table1 (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table1 +POSTHOOK: query: CREATE TABLE table1 (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table1 +PREHOOK: query: explain INSERT INTO table1 values('not', 'null', 'constraint') +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT INTO table1 values('not', 'null', 'constraint') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct('not','null','constraint')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: string), col2 (type: string), col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col0 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: a, b, c + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.table1 + +PREHOOK: query: INSERT INTO table1 values('not', 'null', 'constraint') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@table1 +POSTHOOK: query: INSERT INTO table1 values('not', 'null', 'constraint') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.a SCRIPT [] +POSTHOOK: Lineage: table1.b SCRIPT [] +POSTHOOK: Lineage: table1.c SCRIPT [] +PREHOOK: query: SELECT * FROM table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@table1 +#### A masked pattern was here #### +not null constraint +PREHOOK: query: explain INSERT INTO table1 select key, src.value, value from src +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT INTO table1 select key, src.value, value from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(key is not null) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 22250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: a, b, c + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.table1 + +PREHOOK: query: INSERT INTO table1 select key, src.value, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@table1 +POSTHOOK: query: INSERT INTO table1 select key, src.value, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: table1.c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@table1 +#### A masked pattern was here #### +0 val_0 val_0 +0 val_0 val_0 +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +100 val_100 val_100 +103 val_103 val_103 +103 val_103 val_103 +104 val_104 val_104 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +118 val_118 val_118 +119 val_119 val_119 +119 val_119 val_119 +119 val_119 val_119 +12 val_12 val_12 +12 val_12 val_12 +120 val_120 val_120 +120 val_120 val_120 +125 val_125 val_125 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +129 val_129 val_129 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +137 val_137 val_137 +138 val_138 val_138 +138 val_138 val_138 +138 val_138 val_138 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +146 val_146 val_146 +149 val_149 val_149 +149 val_149 val_149 +15 val_15 val_15 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +164 val_164 val_164 +165 val_165 val_165 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +167 val_167 val_167 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +169 val_169 val_169 +169 val_169 val_169 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +172 val_172 val_172 +174 val_174 val_174 +174 val_174 val_174 +175 val_175 val_175 +175 val_175 val_175 +176 val_176 val_176 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +179 val_179 val_179 +18 val_18 val_18 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +187 val_187 val_187 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +193 val_193 val_193 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +197 val_197 val_197 +199 val_199 val_199 +199 val_199 val_199 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +203 val_203 val_203 +205 val_205 val_205 +205 val_205 val_205 +207 val_207 val_207 +207 val_207 val_207 +208 val_208 val_208 +208 val_208 val_208 +208 val_208 val_208 +209 val_209 val_209 +209 val_209 val_209 +213 val_213 val_213 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +216 val_216 val_216 +217 val_217 val_217 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +219 val_219 val_219 +221 val_221 val_221 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +223 val_223 val_223 +224 val_224 val_224 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +229 val_229 val_229 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +233 val_233 val_233 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +237 val_237 val_237 +238 val_238 val_238 +238 val_238 val_238 +239 val_239 val_239 +239 val_239 val_239 +24 val_24 val_24 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +255 val_255 val_255 +256 val_256 val_256 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +272 val_272 val_272 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +277 val_277 val_277 +277 val_277 val_277 +277 val_277 val_277 +278 val_278 val_278 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +280 val_280 val_280 +281 val_281 val_281 +281 val_281 val_281 +282 val_282 val_282 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +298 val_298 val_298 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +316 val_316 val_316 +316 val_316 val_316 +317 val_317 val_317 +317 val_317 val_317 +318 val_318 val_318 +318 val_318 val_318 +318 val_318 val_318 +321 val_321 val_321 +321 val_321 val_321 +322 val_322 val_322 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +325 val_325 val_325 +327 val_327 val_327 +327 val_327 val_327 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +342 val_342 val_342 +344 val_344 val_344 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +35 val_35 val_35 +35 val_35 val_35 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +37 val_37 val_37 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +382 val_382 val_382 +384 val_384 val_384 +384 val_384 val_384 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +395 val_395 val_395 +396 val_396 val_396 +396 val_396 val_396 +396 val_396 val_396 +397 val_397 val_397 +397 val_397 val_397 +399 val_399 val_399 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +403 val_403 val_403 +403 val_403 val_403 +404 val_404 val_404 +404 val_404 val_404 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +409 val_409 val_409 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +413 val_413 val_413 +414 val_414 val_414 +414 val_414 val_414 +417 val_417 val_417 +417 val_417 val_417 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +430 val_430 val_430 +430 val_430 val_430 +431 val_431 val_431 +431 val_431 val_431 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +438 val_438 val_438 +438 val_438 val_438 +439 val_439 val_439 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +454 val_454 val_454 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +458 val_458 val_458 +459 val_459 val_459 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +462 val_462 val_462 +463 val_463 val_463 +463 val_463 val_463 +466 val_466 val_466 +466 val_466 val_466 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +468 val_468 val_468 +468 val_468 val_468 +468 val_468 val_468 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +480 val_480 val_480 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +489 val_489 val_489 +489 val_489 val_489 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +498 val_498 val_498 +498 val_498 val_498 +5 val_5 val_5 +5 val_5 val_5 +5 val_5 val_5 +51 val_51 val_51 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +70 val_70 val_70 +70 val_70 val_70 +72 val_72 val_72 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +83 val_83 val_83 +84 val_84 val_84 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +90 val_90 val_90 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +97 val_97 val_97 +98 val_98 val_98 +98 val_98 val_98 +not null constraint +PREHOOK: query: explain INSERT OVERWRITE TABLE table1 select src.*, value from src +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT OVERWRITE TABLE table1 select src.*, value from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(key is not null) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 22250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: a, b, c + Statistics: Num rows: 125 Data size: 33625 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.table1 + +PREHOOK: query: INSERT OVERWRITE TABLE table1 select src.*, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@table1 +POSTHOOK: query: INSERT OVERWRITE TABLE table1 select src.*, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: table1.c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@table1 +#### A masked pattern was here #### +0 val_0 val_0 +0 val_0 val_0 +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +100 val_100 val_100 +103 val_103 val_103 +103 val_103 val_103 +104 val_104 val_104 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +118 val_118 val_118 +119 val_119 val_119 +119 val_119 val_119 +119 val_119 val_119 +12 val_12 val_12 +12 val_12 val_12 +120 val_120 val_120 +120 val_120 val_120 +125 val_125 val_125 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +129 val_129 val_129 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +137 val_137 val_137 +138 val_138 val_138 +138 val_138 val_138 +138 val_138 val_138 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +146 val_146 val_146 +149 val_149 val_149 +149 val_149 val_149 +15 val_15 val_15 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +164 val_164 val_164 +165 val_165 val_165 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +167 val_167 val_167 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +169 val_169 val_169 +169 val_169 val_169 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +172 val_172 val_172 +174 val_174 val_174 +174 val_174 val_174 +175 val_175 val_175 +175 val_175 val_175 +176 val_176 val_176 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +179 val_179 val_179 +18 val_18 val_18 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +187 val_187 val_187 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +193 val_193 val_193 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +197 val_197 val_197 +199 val_199 val_199 +199 val_199 val_199 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +203 val_203 val_203 +205 val_205 val_205 +205 val_205 val_205 +207 val_207 val_207 +207 val_207 val_207 +208 val_208 val_208 +208 val_208 val_208 +208 val_208 val_208 +209 val_209 val_209 +209 val_209 val_209 +213 val_213 val_213 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +216 val_216 val_216 +217 val_217 val_217 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +219 val_219 val_219 +221 val_221 val_221 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +223 val_223 val_223 +224 val_224 val_224 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +229 val_229 val_229 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +230 val_230 val_230 +233 val_233 val_233 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +237 val_237 val_237 +238 val_238 val_238 +238 val_238 val_238 +239 val_239 val_239 +239 val_239 val_239 +24 val_24 val_24 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +255 val_255 val_255 +256 val_256 val_256 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +272 val_272 val_272 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +277 val_277 val_277 +277 val_277 val_277 +277 val_277 val_277 +278 val_278 val_278 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +280 val_280 val_280 +281 val_281 val_281 +281 val_281 val_281 +282 val_282 val_282 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +298 val_298 val_298 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +316 val_316 val_316 +316 val_316 val_316 +317 val_317 val_317 +317 val_317 val_317 +318 val_318 val_318 +318 val_318 val_318 +318 val_318 val_318 +321 val_321 val_321 +321 val_321 val_321 +322 val_322 val_322 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +325 val_325 val_325 +327 val_327 val_327 +327 val_327 val_327 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +342 val_342 val_342 +344 val_344 val_344 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +348 val_348 val_348 +35 val_35 val_35 +35 val_35 val_35 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +37 val_37 val_37 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +382 val_382 val_382 +384 val_384 val_384 +384 val_384 val_384 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +395 val_395 val_395 +396 val_396 val_396 +396 val_396 val_396 +396 val_396 val_396 +397 val_397 val_397 +397 val_397 val_397 +399 val_399 val_399 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +403 val_403 val_403 +403 val_403 val_403 +404 val_404 val_404 +404 val_404 val_404 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +409 val_409 val_409 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +413 val_413 val_413 +414 val_414 val_414 +414 val_414 val_414 +417 val_417 val_417 +417 val_417 val_417 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +430 val_430 val_430 +430 val_430 val_430 +431 val_431 val_431 +431 val_431 val_431 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +438 val_438 val_438 +438 val_438 val_438 +439 val_439 val_439 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +454 val_454 val_454 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +458 val_458 val_458 +459 val_459 val_459 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +462 val_462 val_462 +463 val_463 val_463 +463 val_463 val_463 +466 val_466 val_466 +466 val_466 val_466 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +468 val_468 val_468 +468 val_468 val_468 +468 val_468 val_468 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +480 val_480 val_480 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +489 val_489 val_489 +489 val_489 val_489 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +498 val_498 val_498 +498 val_498 val_498 +5 val_5 val_5 +5 val_5 val_5 +5 val_5 val_5 +51 val_51 val_51 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +70 val_70 val_70 +70 val_70 val_70 +72 val_72 val_72 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +83 val_83 val_83 +84 val_84 val_84 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +90 val_90 val_90 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +97 val_97 val_97 +98 val_98 val_98 +98 val_98 val_98 +PREHOOK: query: explain INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(key is not null) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 22250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 33125 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 33125 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: a, b, c + Statistics: Num rows: 125 Data size: 33125 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.table1 + +PREHOOK: query: INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@table1 +POSTHOOK: query: INSERT OVERWRITE TABLE table1 if not exists select src.key, src.key, src.value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: table1.c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@table1 +#### A masked pattern was here #### +0 0 val_0 +0 0 val_0 +0 0 val_0 +10 10 val_10 +100 100 val_100 +100 100 val_100 +103 103 val_103 +103 103 val_103 +104 104 val_104 +104 104 val_104 +105 105 val_105 +11 11 val_11 +111 111 val_111 +113 113 val_113 +113 113 val_113 +114 114 val_114 +116 116 val_116 +118 118 val_118 +118 118 val_118 +119 119 val_119 +119 119 val_119 +119 119 val_119 +12 12 val_12 +12 12 val_12 +120 120 val_120 +120 120 val_120 +125 125 val_125 +125 125 val_125 +126 126 val_126 +128 128 val_128 +128 128 val_128 +128 128 val_128 +129 129 val_129 +129 129 val_129 +131 131 val_131 +133 133 val_133 +134 134 val_134 +134 134 val_134 +136 136 val_136 +137 137 val_137 +137 137 val_137 +138 138 val_138 +138 138 val_138 +138 138 val_138 +138 138 val_138 +143 143 val_143 +145 145 val_145 +146 146 val_146 +146 146 val_146 +149 149 val_149 +149 149 val_149 +15 15 val_15 +15 15 val_15 +150 150 val_150 +152 152 val_152 +152 152 val_152 +153 153 val_153 +155 155 val_155 +156 156 val_156 +157 157 val_157 +158 158 val_158 +160 160 val_160 +162 162 val_162 +163 163 val_163 +164 164 val_164 +164 164 val_164 +165 165 val_165 +165 165 val_165 +166 166 val_166 +167 167 val_167 +167 167 val_167 +167 167 val_167 +168 168 val_168 +169 169 val_169 +169 169 val_169 +169 169 val_169 +169 169 val_169 +17 17 val_17 +170 170 val_170 +172 172 val_172 +172 172 val_172 +174 174 val_174 +174 174 val_174 +175 175 val_175 +175 175 val_175 +176 176 val_176 +176 176 val_176 +177 177 val_177 +178 178 val_178 +179 179 val_179 +179 179 val_179 +18 18 val_18 +18 18 val_18 +180 180 val_180 +181 181 val_181 +183 183 val_183 +186 186 val_186 +187 187 val_187 +187 187 val_187 +187 187 val_187 +189 189 val_189 +19 19 val_19 +190 190 val_190 +191 191 val_191 +191 191 val_191 +192 192 val_192 +193 193 val_193 +193 193 val_193 +193 193 val_193 +194 194 val_194 +195 195 val_195 +195 195 val_195 +196 196 val_196 +197 197 val_197 +197 197 val_197 +199 199 val_199 +199 199 val_199 +199 199 val_199 +2 2 val_2 +20 20 val_20 +200 200 val_200 +200 200 val_200 +201 201 val_201 +202 202 val_202 +203 203 val_203 +203 203 val_203 +205 205 val_205 +205 205 val_205 +207 207 val_207 +207 207 val_207 +208 208 val_208 +208 208 val_208 +208 208 val_208 +209 209 val_209 +209 209 val_209 +213 213 val_213 +213 213 val_213 +214 214 val_214 +216 216 val_216 +216 216 val_216 +217 217 val_217 +217 217 val_217 +218 218 val_218 +219 219 val_219 +219 219 val_219 +221 221 val_221 +221 221 val_221 +222 222 val_222 +223 223 val_223 +223 223 val_223 +224 224 val_224 +224 224 val_224 +226 226 val_226 +228 228 val_228 +229 229 val_229 +229 229 val_229 +230 230 val_230 +230 230 val_230 +230 230 val_230 +230 230 val_230 +230 230 val_230 +233 233 val_233 +233 233 val_233 +235 235 val_235 +237 237 val_237 +237 237 val_237 +238 238 val_238 +238 238 val_238 +239 239 val_239 +239 239 val_239 +24 24 val_24 +24 24 val_24 +241 241 val_241 +242 242 val_242 +242 242 val_242 +244 244 val_244 +247 247 val_247 +248 248 val_248 +249 249 val_249 +252 252 val_252 +255 255 val_255 +255 255 val_255 +256 256 val_256 +256 256 val_256 +257 257 val_257 +258 258 val_258 +26 26 val_26 +26 26 val_26 +260 260 val_260 +262 262 val_262 +263 263 val_263 +265 265 val_265 +265 265 val_265 +266 266 val_266 +27 27 val_27 +272 272 val_272 +272 272 val_272 +273 273 val_273 +273 273 val_273 +273 273 val_273 +274 274 val_274 +275 275 val_275 +277 277 val_277 +277 277 val_277 +277 277 val_277 +277 277 val_277 +278 278 val_278 +278 278 val_278 +28 28 val_28 +280 280 val_280 +280 280 val_280 +281 281 val_281 +281 281 val_281 +282 282 val_282 +282 282 val_282 +283 283 val_283 +284 284 val_284 +285 285 val_285 +286 286 val_286 +287 287 val_287 +288 288 val_288 +288 288 val_288 +289 289 val_289 +291 291 val_291 +292 292 val_292 +296 296 val_296 +298 298 val_298 +298 298 val_298 +298 298 val_298 +30 30 val_30 +302 302 val_302 +305 305 val_305 +306 306 val_306 +307 307 val_307 +307 307 val_307 +308 308 val_308 +309 309 val_309 +309 309 val_309 +310 310 val_310 +311 311 val_311 +311 311 val_311 +311 311 val_311 +315 315 val_315 +316 316 val_316 +316 316 val_316 +316 316 val_316 +317 317 val_317 +317 317 val_317 +318 318 val_318 +318 318 val_318 +318 318 val_318 +321 321 val_321 +321 321 val_321 +322 322 val_322 +322 322 val_322 +323 323 val_323 +325 325 val_325 +325 325 val_325 +327 327 val_327 +327 327 val_327 +327 327 val_327 +33 33 val_33 +331 331 val_331 +331 331 val_331 +332 332 val_332 +333 333 val_333 +333 333 val_333 +335 335 val_335 +336 336 val_336 +338 338 val_338 +339 339 val_339 +34 34 val_34 +341 341 val_341 +342 342 val_342 +342 342 val_342 +344 344 val_344 +344 344 val_344 +345 345 val_345 +348 348 val_348 +348 348 val_348 +348 348 val_348 +348 348 val_348 +348 348 val_348 +35 35 val_35 +35 35 val_35 +35 35 val_35 +351 351 val_351 +353 353 val_353 +353 353 val_353 +356 356 val_356 +360 360 val_360 +362 362 val_362 +364 364 val_364 +365 365 val_365 +366 366 val_366 +367 367 val_367 +367 367 val_367 +368 368 val_368 +369 369 val_369 +369 369 val_369 +369 369 val_369 +37 37 val_37 +37 37 val_37 +373 373 val_373 +374 374 val_374 +375 375 val_375 +377 377 val_377 +378 378 val_378 +379 379 val_379 +382 382 val_382 +382 382 val_382 +384 384 val_384 +384 384 val_384 +384 384 val_384 +386 386 val_386 +389 389 val_389 +392 392 val_392 +393 393 val_393 +394 394 val_394 +395 395 val_395 +395 395 val_395 +396 396 val_396 +396 396 val_396 +396 396 val_396 +397 397 val_397 +397 397 val_397 +399 399 val_399 +399 399 val_399 +4 4 val_4 +400 400 val_400 +401 401 val_401 +401 401 val_401 +401 401 val_401 +401 401 val_401 +401 401 val_401 +402 402 val_402 +403 403 val_403 +403 403 val_403 +403 403 val_403 +404 404 val_404 +404 404 val_404 +406 406 val_406 +406 406 val_406 +406 406 val_406 +406 406 val_406 +407 407 val_407 +409 409 val_409 +409 409 val_409 +409 409 val_409 +41 41 val_41 +411 411 val_411 +413 413 val_413 +413 413 val_413 +414 414 val_414 +414 414 val_414 +417 417 val_417 +417 417 val_417 +417 417 val_417 +418 418 val_418 +419 419 val_419 +42 42 val_42 +42 42 val_42 +421 421 val_421 +424 424 val_424 +424 424 val_424 +427 427 val_427 +429 429 val_429 +429 429 val_429 +43 43 val_43 +430 430 val_430 +430 430 val_430 +430 430 val_430 +431 431 val_431 +431 431 val_431 +431 431 val_431 +432 432 val_432 +435 435 val_435 +436 436 val_436 +437 437 val_437 +438 438 val_438 +438 438 val_438 +438 438 val_438 +439 439 val_439 +439 439 val_439 +44 44 val_44 +443 443 val_443 +444 444 val_444 +446 446 val_446 +448 448 val_448 +449 449 val_449 +452 452 val_452 +453 453 val_453 +454 454 val_454 +454 454 val_454 +454 454 val_454 +455 455 val_455 +457 457 val_457 +458 458 val_458 +458 458 val_458 +459 459 val_459 +459 459 val_459 +460 460 val_460 +462 462 val_462 +462 462 val_462 +463 463 val_463 +463 463 val_463 +466 466 val_466 +466 466 val_466 +466 466 val_466 +467 467 val_467 +468 468 val_468 +468 468 val_468 +468 468 val_468 +468 468 val_468 +469 469 val_469 +469 469 val_469 +469 469 val_469 +469 469 val_469 +469 469 val_469 +47 47 val_47 +470 470 val_470 +472 472 val_472 +475 475 val_475 +477 477 val_477 +478 478 val_478 +478 478 val_478 +479 479 val_479 +480 480 val_480 +480 480 val_480 +480 480 val_480 +481 481 val_481 +482 482 val_482 +483 483 val_483 +484 484 val_484 +485 485 val_485 +487 487 val_487 +489 489 val_489 +489 489 val_489 +489 489 val_489 +489 489 val_489 +490 490 val_490 +491 491 val_491 +492 492 val_492 +492 492 val_492 +493 493 val_493 +494 494 val_494 +495 495 val_495 +496 496 val_496 +497 497 val_497 +498 498 val_498 +498 498 val_498 +498 498 val_498 +5 5 val_5 +5 5 val_5 +5 5 val_5 +51 51 val_51 +51 51 val_51 +53 53 val_53 +54 54 val_54 +57 57 val_57 +58 58 val_58 +58 58 val_58 +64 64 val_64 +65 65 val_65 +66 66 val_66 +67 67 val_67 +67 67 val_67 +69 69 val_69 +70 70 val_70 +70 70 val_70 +70 70 val_70 +72 72 val_72 +72 72 val_72 +74 74 val_74 +76 76 val_76 +76 76 val_76 +77 77 val_77 +78 78 val_78 +8 8 val_8 +80 80 val_80 +82 82 val_82 +83 83 val_83 +83 83 val_83 +84 84 val_84 +84 84 val_84 +85 85 val_85 +86 86 val_86 +87 87 val_87 +9 9 val_9 +90 90 val_90 +90 90 val_90 +90 90 val_90 +92 92 val_92 +95 95 val_95 +95 95 val_95 +96 96 val_96 +97 97 val_97 +97 97 val_97 +98 98 val_98 +98 98 val_98 +PREHOOK: query: DROP TABLE table1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table1 +PREHOOK: Output: default@table1 +POSTHOOK: query: DROP TABLE table1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table1 +POSTHOOK: Output: default@table1 +PREHOOK: query: create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 (a STRING NOT NULL ENFORCED, b STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: create table src_multi2 (i STRING, j STRING NOT NULL ENABLE) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 (i STRING, j STRING NOT NULL ENABLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: explain +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and enforce_constraint(key is not null)) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: a, b + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Filter Operator + predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: i, j + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b + Column Types: string, string + Table: default.src_multi1 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: string, string + Table: default.src_multi2 + +PREHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.i SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.j SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and enforce_constraint(key is not null)) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: a, b + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Filter Operator + predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: i, j + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b + Column Types: string, string + Table: default.src_multi1 + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: string, string + Table: default.src_multi2 + +PREHOOK: query: from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: from src +insert into table src_multi1 select * where src.key < 10 +insert into table src_multi2 select src.* where key > 10 and key < 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.i SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.j SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table acid_uami(i int, + de decimal(5,2) constraint nn1 not null enforced, + vc varchar(128) constraint nn2 not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uami +POSTHOOK: query: create table acid_uami(i int, + de decimal(5,2) constraint nn1 not null enforced, + vc varchar(128) constraint nn2 not null enforced) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uami +PREHOOK: query: explain insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(1,109.23,'mary had a little lamb'),const struct(6553,923.19,'its fleece was white as snow')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: decimal(5,2)), col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de SCRIPT [] +POSTHOOK: Lineage: acid_uami.i SCRIPT [] +POSTHOOK: Lineage: acid_uami.vc SCRIPT [] +PREHOOK: query: select * from acid_uami +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uami +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +#### A masked pattern was here #### +1 109.23 mary had a little lamb +6553 923.19 its fleece was white as snow +PREHOOK: query: --insert into select +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +PREHOOK: type: QUERY +POSTHOOK: query: --insert into select +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 25875 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 125 Data size: 25875 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src limit 2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + order by key limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + order by key limit 2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + group by key, value order by key limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src + group by key, value order by key limit 2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), CAST( _col0 AS decimal(5,2)) (type: decimal(5,2)), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 51750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 414 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 207 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: --overwrite +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +PREHOOK: type: QUERY +POSTHOOK: query: --overwrite +explain insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col1 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 25875 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 125 Data size: 25875 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 125 Data size: 41000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + +PREHOOK: query: insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami select cast(key as int), cast (key as decimal(5,2)), value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: acid_uami + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((de = 109.23) or (de = 119.23)) and enforce_constraint(vc is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ROW__ID (type: struct), i (type: int), vc (type: varchar(128)) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: varchar(128)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: UPDATE + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: UPDATE + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: ALTER table acid_uami drop constraint nn1 +PREHOOK: type: ALTERTABLE_DROPCONSTRAINT +POSTHOOK: query: ALTER table acid_uami drop constraint nn1 +POSTHOOK: type: ALTERTABLE_DROPCONSTRAINT +PREHOOK: query: ALTER table acid_uami CHANGE i i int constraint nn0 not null enforced +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: ALTER table acid_uami CHANGE i i int constraint nn0 not null enforced +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: explain update acid_uami set de = 3.14159 where de = 3.14 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid_uami set de = 3.14159 where de = 3.14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: acid_uami + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((de = 3.14) and enforce_constraint(i is not null) and enforce_constraint(vc is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ROW__ID (type: struct), i (type: int), vc (type: varchar(128)) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: varchar(128)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: UPDATE + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: UPDATE + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: update acid_uami set de = 3.14159 where de = 3.14 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: update acid_uami set de = 3.14159 where de = 3.14 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: explain +from src +insert overwrite table acid_uami select cast(key as int), cast(key as decimal(5,2)), value where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from src +insert overwrite table acid_uami select cast(key as int), cast(key as decimal(5,2)), value where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: enforce_constraint(_col0 is not null) (type: boolean) + Statistics: Num rows: 41 Data size: 8487 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 41 Data size: 8487 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + Filter Operator + predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: i, j + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: decimal(5,2)), CAST( VALUE._col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 41 Data size: 13448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 41 Data size: 13448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) + outputColumnNames: i, de, vc + Statistics: Num rows: 41 Data size: 13448 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(de, 'hll'), compute_stats(vc, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1496 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid_uami + Write Type: INSERT + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, de, vc + Column Types: int, decimal(5,2), varchar(128) + Table: default.acid_uami + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: string, string + Table: default.src_multi2 + +PREHOOK: query: CREATE TABLE tablePartitioned (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED) + PARTITIONED BY (p1 STRING, p2 INT NOT NULL ENABLE) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tablePartitioned +POSTHOOK: query: CREATE TABLE tablePartitioned (a STRING NOT NULL ENFORCED, b STRING, c STRING NOT NULL ENFORCED) + PARTITIONED BY (p1 STRING, p2 INT NOT NULL ENABLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tablePartitioned +PREHOOK: query: explain INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint') +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct('not','null','constraint')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: string), col2 (type: string), col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col0 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), 'today' (type: string), UDFToInteger('10') (type: int) + outputColumnNames: a, b, c, p1, p2 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + keys: p1 (type: string), p2 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1413 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 1 Data size: 1413 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1413 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1413 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1413 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + p1 today + p2 10 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.tablepartitioned + +PREHOOK: query: INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tablepartitioned@p1=today/p2=10 +POSTHOOK: query: INSERT INTO tablePartitioned partition(p1='today', p2=10) values('not', 'null', 'constraint') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tablepartitioned@p1=today/p2=10 +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=today,p2=10).a SCRIPT [] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=today,p2=10).b SCRIPT [] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=today,p2=10).c SCRIPT [] +PREHOOK: query: explain INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), 3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(_col0 is not null) and enforce_constraint(_col2 is not null)) (type: boolean) + Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: a, b, c, p1 + Statistics: Num rows: 2 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + keys: p1 (type: string), 3 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1411 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), 3 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), 3 (type: int) + Statistics: Num rows: 1 Data size: 1411 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + keys: KEY._col0 (type: string), 3 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1411 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: string), 3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1411 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1411 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + p1 + p2 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.tablepartitioned + +PREHOOK: query: INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tablepartitioned +POSTHOOK: query: INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, key as p1, 3 as p2 from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tablepartitioned@p1=165/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=238/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=255/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=27/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=278/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=311/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=409/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=484/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=86/p2=3 +POSTHOOK: Output: default@tablepartitioned@p1=98/p2=3 +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=165,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=165,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=165,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=238,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=238,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=238,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=255,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=255,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=255,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=27,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=27,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=27,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=278,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=278,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=278,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=311,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=311,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=311,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=409,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=409,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=409,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=484,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=484,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=484,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=86,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=86,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=86,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=98,p2=3).a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=98,p2=3).b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tablepartitioned PARTITION(p1=98,p2=3).c SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tablePartitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@tablepartitioned +PREHOOK: Input: default@tablepartitioned@p1=165/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=238/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=255/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=27/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=278/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=311/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=409/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=484/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=86/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=98/p2=3 +PREHOOK: Input: default@tablepartitioned@p1=today/p2=10 +#### A masked pattern was here #### +POSTHOOK: query: select * from tablePartitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tablepartitioned +POSTHOOK: Input: default@tablepartitioned@p1=165/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=238/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=255/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=27/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=278/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=311/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=409/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=484/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=86/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=98/p2=3 +POSTHOOK: Input: default@tablepartitioned@p1=today/p2=10 +#### A masked pattern was here #### +165 val_165 val_165 165 3 +238 val_238 val_238 238 3 +255 val_255 val_255 255 3 +27 val_27 val_27 27 3 +278 val_278 val_278 278 3 +311 val_311 val_311 311 3 +409 val_409 val_409 409 3 +484 val_484 val_484 484 3 +86 val_86 val_86 86 3 +98 val_98 val_98 98 3 +not null constraint today 10 +PREHOOK: query: explain +from src +INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, 'yesterday' as p1, 3 as p2 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from src +INSERT INTO tablePartitioned partition(p1, p2) select key, value, value, 'yesterday' as p1, 3 as p2 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), 'yesterday' (type: string), 3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (enforce_constraint(key is not null) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 125 Data size: 45750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 45750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: a, b, c + Statistics: Num rows: 125 Data size: 45750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll'), compute_stats(c, 'hll') + keys: 'yesterday' (type: string), 3 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1417 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 'yesterday' (type: string), 3 (type: int) + sort order: ++ + Map-reduce partition columns: 'yesterday' (type: string), 3 (type: int) + Statistics: Num rows: 1 Data size: 1417 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Filter Operator + predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: i, j + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + keys: 'yesterday' (type: string), 3 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1417 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), 'yesterday' (type: string), 3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1417 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1417 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + p1 + p2 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tablepartitioned + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b, c + Column Types: string, string, string + Table: default.tablepartitioned + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: string, string + Table: default.src_multi2 + +PREHOOK: query: DROP TABLE src_multi1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_multi1 +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: DROP TABLE src_multi1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_multi1 +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: DROP TABLE src_multi2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_multi2 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: DROP TABLE src_multi2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_multi2 +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: DROP TABLE acid_uami +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: DROP TABLE acid_uami +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: create table nonacid (key int, a1 string, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nonacid +POSTHOOK: query: create table nonacid (key int, a1 string, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nonacid +PREHOOK: query: create table masking_test (key int NOT NULL enable, a1 string, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test (key int NOT NULL enable, a1 string, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string), ROW__ID (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + value expressions: a1 (type: string), value (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col2, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 5)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 3) and (_col6 >= 5) and enforce_constraint(_col0 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct), _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: string) + Filter Operator + predicate: (_col0 is null and enforce_constraint(_col6 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), '1' (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, a1, value + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + + Stage: Stage-5 + Stats Work + Basic Stats Work: + + Stage: Stage-2 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + + Stage: Stage-6 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + + Stage: Stage-7 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, a1, value + Column Types: int, string, string + Table: default.masking_test + +PREHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-5 + Stage-6 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-5 + Stage-7 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-5 + Stage-8 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-5 + Stage-9 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (SIMPLE_EDGE) + Reducer 6 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string), ROW__ID (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + value expressions: a1 (type: string), value (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col2, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 5)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 3) and (_col6 >= 5) and enforce_constraint(_col0 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct), _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: string) + Filter Operator + predicate: (_col0 = _col6) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col5 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col5 (type: struct) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: _col0 (type: struct) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Filter Operator + predicate: (_col0 is null and enforce_constraint(_col6 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), '1' (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 > 1) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cardinality_violation(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + Select Operator + expressions: _col0 (type: int) + outputColumnNames: val + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(val, 'hll') + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, a1, value + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + + Stage: Stage-6 + Stats Work + Basic Stats Work: + + Stage: Stage-2 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + + Stage: Stage-7 + Stats Work + Basic Stats Work: + + Stage: Stage-3 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + + Stage: Stage-8 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: val + Column Types: int + Table: default.merge_tmp_table + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + + Stage: Stage-9 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, a1, value + Column Types: int, string, string + Table: default.masking_test + +PREHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: ROW__ID (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + value expressions: a1 (type: string), value (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 5)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = _col6) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col5 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col5 (type: struct) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: _col0 (type: struct) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Filter Operator + predicate: (_col0 is null and enforce_constraint(_col6 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 > 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cardinality_violation(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + Select Operator + expressions: _col0 (type: int) + outputColumnNames: val + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(val, 'hll') + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, a1, value + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + + Stage: Stage-5 + Stats Work + Basic Stats Work: + + Stage: Stage-2 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + + Stage: Stage-6 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: val + Column Types: int + Table: default.merge_tmp_table + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + + Stage: Stage-7 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, a1, value + Column Types: int, string, string + Table: default.masking_test + +PREHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 3 THEN UPDATE set a1 = '1' +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.a1, s.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-4 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-3 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string), ROW__ID (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + value expressions: a1 (type: string), value (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col2, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 3) and enforce_constraint(_col0 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct), _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: string) + Filter Operator + predicate: (_col0 = _col6) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col5 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col5 (type: struct) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: _col0 (type: struct) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Filter Operator + predicate: (_col0 is null and enforce_constraint(_col6 is not null)) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), '1' (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 > 1) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cardinality_violation(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + Select Operator + expressions: _col0 (type: int) + outputColumnNames: val + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(val, 'hll') + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, a1, value + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Dependency Collection + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: UPDATE + + Stage: Stage-5 + Stats Work + Basic Stats Work: + + Stage: Stage-2 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + + Stage: Stage-6 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: val + Column Types: int + Table: default.merge_tmp_table + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: INSERT + + Stage: Stage-7 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, a1, value + Column Types: int, string, string + Table: default.masking_test + +PREHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +PREHOOK: type: QUERY +POSTHOOK: query: explain MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: ROW__ID (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col5, _col6 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col6) and (_col6 < 5)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = _col6) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: struct) + outputColumnNames: _col5 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col5 (type: struct) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: _col0 (type: struct) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 > 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cardinality_violation(_col0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + Select Operator + expressions: _col0 (type: int) + outputColumnNames: val + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(val, 'hll') + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test + Write Type: DELETE + + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_tmp_table + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: val + Column Types: int + Table: default.merge_tmp_table + +PREHOOK: query: DROP TABLE masking_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: DROP TABLE masking_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: DROP TABLE nonacid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nonacid +PREHOOK: Output: default@nonacid +POSTHOOK: query: DROP TABLE nonacid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nonacid +POSTHOOK: Output: default@nonacid +PREHOOK: query: create table table2(i int constraint nn5 not null enforced, j int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table2 +POSTHOOK: query: create table table2(i int constraint nn5 not null enforced, j int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table2 +PREHOOK: query: explain insert into table2 values(2, 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table2 values(2, 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(2,3)) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: enforce_constraint(_col0 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table2 + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: i, j + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table2 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: int, int + Table: default.table2 + +PREHOOK: query: alter table table2 drop constraint nn5 +PREHOOK: type: ALTERTABLE_DROPCONSTRAINT +POSTHOOK: query: alter table table2 drop constraint nn5 +POSTHOOK: type: ALTERTABLE_DROPCONSTRAINT +PREHOOK: query: explain insert into table2 values(2, 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table2 values(2, 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(2,3)) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table2 + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: i, j + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll'), compute_stats(j, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.table2 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i, j + Column Types: int, int + Table: default.table2 + +PREHOOK: query: DROP TABLE table2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table2 +PREHOOK: Output: default@table2 +POSTHOOK: query: DROP TABLE table2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table2 +POSTHOOK: Output: default@table2 +PREHOOK: query: create temporary table tttemp(i int not null enforced) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tttemp +POSTHOOK: query: create temporary table tttemp(i int not null enforced) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tttemp +PREHOOK: query: explain insert into tttemp values(1) +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into tttemp values(1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(1)) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: enforce_constraint(_col0 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tttemp + Select Operator + expressions: _col0 (type: int) + outputColumnNames: i + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll') + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tttemp + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i + Column Types: int + Table: default.tttemp + +PREHOOK: query: explain insert into tttemp select cast(key as int) from src +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into tttemp select cast(key as int) from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: enforce_constraint(_col0 is not null) (type: boolean) + Statistics: Num rows: 250 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tttemp + Select Operator + expressions: _col0 (type: int) + outputColumnNames: i + Statistics: Num rows: 250 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(i, 'hll') + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tttemp + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: i + Column Types: int + Table: default.tttemp + +PREHOOK: query: drop table tttemp +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tttemp +PREHOOK: Output: default@tttemp +POSTHOOK: query: drop table tttemp +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tttemp +POSTHOOK: Output: default@tttemp diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out index 43e4a5de39..fcbf2c53cc 100644 --- a/ql/src/test/results/clientpositive/show_functions.q.out +++ b/ql/src/test/results/clientpositive/show_functions.q.out @@ -81,6 +81,7 @@ div e elt encode +enforce_constraint ewah_bitmap ewah_bitmap_and ewah_bitmap_empty