diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index fd0058a..44a491a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -17,6 +17,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -69,6 +70,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; @@ -92,6 +97,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -219,6 +225,65 @@ private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map grandChildren = child.getChildren(); + + if (childUDF instanceof GenericUDFBaseCompare || + childUDF instanceof GenericUDFOPNull || + childUDF instanceof GenericUDFOPNotNull) { + List newGrandChildren = new ArrayList(); + for(ExprNodeDesc grandChild : grandChildren) { + newGrandChildren.add(foldNegative(grandChild)); + } + + return ExprNodeGenericFuncDesc.newInstance( + childUDF.negative(), + newGrandChildren); + } else if (childUDF instanceof GenericUDFOPAnd || + childUDF instanceof GenericUDFOPOr) { + List newGrandChildren = new ArrayList(); + for(ExprNodeDesc grandChild : grandChildren) { + newGrandChildren.add(foldNegative( + ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), + Arrays.asList(grandChild)))); + } + + return ExprNodeGenericFuncDesc.newInstance( + childUDF.negative(), + newGrandChildren); + }else if (childUDF instanceof GenericUDFOPNot) { + return foldNegative(child.getChildren().get(0)); + } else { + // For operator like if() that cannot be handled, leave not() as it + // is and continue processing the children + List newGrandChildren = new ArrayList(); + for(ExprNodeDesc grandChild : grandChildren) { + newGrandChildren.add(foldNegative(grandChild)); + } + childDesc.setChildren(newGrandChildren); + return funcDesc; + } + } + } + } + return desc; + } + + /** * Fold input expression desc, only performing short-cutting. * * Unnecessary AND/OR operations involving a constant true/false value will be eliminated. @@ -234,6 +299,11 @@ private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map constants, ConstantPropagateProcCtx cppCtx, Operator op, int tag, boolean propagate) throws UDFArgumentException { + // Combine NOT operator with the child operator. Otherwise, the following optimization + // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null), + // should not be optimized to not(x > 3), but (x <=3 or x is null). + desc = foldNegative(desc); + if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; @@ -245,9 +315,11 @@ private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map 200112 for year=2001, month=12 since concat is deterministic UDF - * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nonderministic UDF + * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nondeterministic UDF * @param desc folding expression * @param constants current propagated constant map * @param cppCtx @@ -284,6 +356,11 @@ private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map constants, ConstantPropagateProcCtx cppCtx, Operator op, int tag, boolean propagate) throws UDFArgumentException { + // Combine NOT operator with the child operator. Otherwise, the following optimization + // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null), + // should not be optimized to not(x > 3), but (x <=3 or x is null). + desc = foldNegative(desc); + if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; @@ -295,9 +372,11 @@ private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map newExprs, Operator op) throws UDFArgumentException { - if (udf instanceof GenericUDFOPEqual) { assert newExprs.size() == 2; boolean foundUDFInFirst = false; @@ -491,6 +569,7 @@ private static ExprNodeDesc shortcutFunction(GenericUDF udf, List return null; } } + if (udf instanceof GenericUDFOPAnd) { for (int i = 0; i < 2; i++) { ExprNodeDesc childExpr = newExprs.get(i); @@ -506,12 +585,15 @@ private static ExprNodeDesc shortcutFunction(GenericUDF udf, List // if false return false return childExpr; } - } else // Try to fold (key = 86) and (key is not null) to (key = 86) - if (childExpr instanceof ExprNodeGenericFuncDesc && + } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && - childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc && other instanceof ExprNodeGenericFuncDesc - && ((ExprNodeGenericFuncDesc)other).getGenericUDF() instanceof GenericUDFBaseCompare - && other.getChildren().size() == 2) { + childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc && other instanceof ExprNodeGenericFuncDesc && + ((ExprNodeGenericFuncDesc)other).getGenericUDF() instanceof GenericUDFBaseCompare && + !(((ExprNodeGenericFuncDesc)other).getGenericUDF() instanceof GenericUDFOPNotEqual) && + other.getChildren().size() == 2) { + // Try to fold (key 86) and (key is not null) to (key 86) + // where can be "=", ">=", "<=", ">", "<". + // Note: (key <> 86) and (key is not null) cannot be folded ExprNodeColumnDesc colDesc = getColumnExpr(other.getChildren().get(0)); if (null == colDesc) { colDesc = getColumnExpr(other.getChildren().get(1)); @@ -587,6 +669,7 @@ private static ExprNodeDesc shortcutFunction(GenericUDF udf, List } } } + if (udf instanceof GenericUDFCase) { // HIVE-9644 Attempt to fold expression like : // where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java index e8df5d3..259fde8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java @@ -213,12 +213,24 @@ public void close() throws IOException { } /** - * Some functions are affected by appearing order of arguments (comparisons, for example) + * Some functions like comparisons may be affected by appearing order of arguments. + * This is to convert a function, such as 3 > x to x < 3. The flip function of + * GenericUDFOPGreaterThan is GenericUDFOPLessThan. */ public GenericUDF flip() { return this; } + /** + * Gets the negative function of the current one. E.g., GenericUDFOPNotEqual for + * GenericUDFOPEqual, or GenericUDFOPNull for GenericUDFOPNotNull. + * @return Negative function + * @throws UDFArgumentException + */ + public GenericUDF negative() { + throw new UnsupportedOperationException("Negative function doesn't exist for " + getFuncName()); + } + public String getUdfName() { return getClass().getName(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java index 47abb20..459b63b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java @@ -88,4 +88,8 @@ public String getDisplayString(String[] children) { return "(" + children[0] + " and " + children[1] + ")"; } + @Override + public GenericUDF negative() { + return new GenericUDFOPOr(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java index 428ced7..23f7f1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java @@ -122,4 +122,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { return result; } + @Override + public GenericUDF negative() { + return new GenericUDFOPNotEqual(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java index d9556cc..bd5b61a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java @@ -134,4 +134,8 @@ public GenericUDF flip() { return new GenericUDFOPEqualOrLessThan(); } + @Override + public GenericUDF negative() { + return new GenericUDFOPLessThan(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java index 1d9eaf6..ad971cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java @@ -134,5 +134,9 @@ public GenericUDF flip() { return new GenericUDFOPEqualOrGreaterThan(); } + @Override + public GenericUDF negative() { + return new GenericUDFOPGreaterThan(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java index 8e1f2b1..85f0601 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java @@ -134,5 +134,9 @@ public GenericUDF flip() { return new GenericUDFOPLessThan(); } + @Override + public GenericUDF negative() { + return new GenericUDFOPEqualOrLessThan(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java index 101b348..b445ae6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java @@ -134,4 +134,8 @@ public GenericUDF flip() { return new GenericUDFOPGreaterThan(); } + @Override + public GenericUDF negative() { + return new GenericUDFOPEqualOrGreaterThan(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java index b5da57a..ac8ec2f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java @@ -121,4 +121,9 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { } return result; } + + @Override + public GenericUDF negative() { + return new GenericUDFOPEqual(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java index d22b35d..2b67c38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java @@ -60,4 +60,8 @@ public String getDisplayString(String[] children) { return children[0] + " is not null"; } + @Override + public GenericUDF negative() { + return new GenericUDFOPNull(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java index fe20e9a..4eb92eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java @@ -60,4 +60,8 @@ public String getDisplayString(String[] children) { return children[0] + " is null"; } + @Override + public GenericUDF negative() { + return new GenericUDFOPNotNull(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java index cd656a0..cbe04eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java @@ -89,4 +89,8 @@ public String getDisplayString(String[] children) { return "(" + children[0] + " or " + children[1] + ")"; } + @Override + public GenericUDF negative() { + return new GenericUDFOPAnd(); + } } diff --git a/ql/src/test/queries/clientpositive/folder_predicate.q b/ql/src/test/queries/clientpositive/folder_predicate.q new file mode 100644 index 0000000..2377dd4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/folder_predicate.q @@ -0,0 +1,32 @@ +drop table if exists predicate_fold_tb; + +create table predicate_fold_tb(value int); +insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3); + +explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3); +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3); diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out index 492e302..054b573 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -297,15 +297,15 @@ STAGE PLANS: alias: loc_orc Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not zip is not null) (type: boolean) + predicate: zip is null (type: boolean) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -375,7 +375,7 @@ STAGE PLANS: alias: loc_orc Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not zip is null) (type: boolean) + predicate: zip is not null (type: boolean) Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out index f36670c..338b754 100644 --- a/ql/src/test/results/clientpositive/decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/decimal_udf.q.out @@ -973,12 +973,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key <> 0) (type: boolean) - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key <> 0)) (type: boolean) + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / key) (type: decimal(38,24)) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 @@ -1039,12 +1039,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21)) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 @@ -1095,12 +1095,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out index e09057a..af42d5c 100644 --- a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out +++ b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out @@ -37,7 +37,7 @@ STAGE PLANS: alias: f Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value <> '') and key is not null) and value is not null) (type: boolean) + predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -86,17 +86,17 @@ STAGE PLANS: alias: f Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> '') (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> '')) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -105,14 +105,14 @@ STAGE PLANS: 0 _col3 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col4 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -163,7 +163,7 @@ STAGE PLANS: alias: f Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value <> '') and key is not null) and value is not null) (type: boolean) + predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -212,17 +212,17 @@ STAGE PLANS: alias: f Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> '') (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> '')) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -231,14 +231,14 @@ STAGE PLANS: 0 _col3 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col4 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/fold_when.q.out b/ql/src/test/results/clientpositive/fold_when.q.out index 0a4b328..31b4f6a 100644 --- a/ql/src/test/results/clientpositive/fold_when.q.out +++ b/ql/src/test/results/clientpositive/fold_when.q.out @@ -328,15 +328,15 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not (key = '238')) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (key <> '238') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -435,15 +435,15 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not (key = '11')) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (key <> '11') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/folder_predicate.q.out b/ql/src/test/results/clientpositive/folder_predicate.q.out new file mode 100644 index 0000000..fa27412 --- /dev/null +++ b/ql/src/test/results/clientpositive/folder_predicate.q.out @@ -0,0 +1,368 @@ +PREHOOK: query: drop table if exists predicate_fold_tb +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists predicate_fold_tb +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table predicate_fold_tb(value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@predicate_fold_tb +POSTHOOK: query: create table predicate_fold_tb(value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@predicate_fold_tb +PREHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@predicate_fold_tb +POSTHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@predicate_fold_tb +POSTHOOK: Lineage: predicate_fold_tb.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value <> 3)) (type: boolean) + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +1 +2 +4 +5 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value < 3)) (type: boolean) + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +1 +2 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value > 3)) (type: boolean) + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +4 +5 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value <= 3)) (type: boolean) + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +1 +2 +3 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value >= 3)) (type: boolean) + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +3 +4 +5 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or (value = 3)) (type: boolean) + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +3 +PREHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: predicate_fold_tb + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is null or ((value <= 1) or (value > 3))) (type: boolean) + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3) +PREHOOK: type: QUERY +PREHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@predicate_fold_tb +#### A masked pattern was here #### +NULL +1 +4 +5 diff --git a/ql/src/test/results/clientpositive/input_testxpath2.q.out b/ql/src/test/results/clientpositive/input_testxpath2.q.out index d3a6f29..67e270f 100644 --- a/ql/src/test/results/clientpositive/input_testxpath2.q.out +++ b/ql/src/test/results/clientpositive/input_testxpath2.q.out @@ -32,7 +32,7 @@ STAGE PLANS: alias: src_thrift Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (lint is not null and (not mstringstring is null)) (type: boolean) + predicate: (lint is not null and mstringstring is not null) (type: boolean) Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int) diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out index d0eb853..073f061 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out @@ -295,12 +295,12 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (not (x = 86)) (type: boolean) - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE + predicate: (x <> 86) (type: boolean) + Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: x (type: int) outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- List Bucketing Query diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out index ef8ee2f..634e171 100644 --- a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out +++ b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out @@ -115,12 +115,12 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((rand(1) < 0.1) and (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0)))) (type: boolean) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2' @@ -247,12 +247,12 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0))) (type: boolean) - Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE + predicate: ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select a.* from srcpart a where a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2' diff --git a/ql/src/test/results/clientpositive/select_unquote_not.q.out b/ql/src/test/results/clientpositive/select_unquote_not.q.out index 64de433..3b8146b 100644 --- a/ql/src/test/results/clientpositive/select_unquote_not.q.out +++ b/ql/src/test/results/clientpositive/select_unquote_not.q.out @@ -59,15 +59,15 @@ STAGE PLANS: alias: npe_test Statistics: Num rows: 498 Data size: 5290 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not (UDFToDouble(ds) < 1970.0)) (type: boolean) - Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(ds) >= 1970.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out index 06f1ca4..99120bb 100644 --- a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out +++ b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out @@ -289,13 +289,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (value <> '') (type: boolean) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + predicate: ((value <> '') and value is not null) (type: boolean) + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true Path -> Alias: @@ -379,17 +379,17 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0, _col13 Position of Big Table: 0 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col13 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out index b24ebcc..13f0de2 100644 --- a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out @@ -1239,15 +1239,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / key) (type: decimal(38,24)) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1321,15 +1321,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21)) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1393,15 +1393,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out index d459ce3..b59a5b9 100644 --- a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out +++ b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out @@ -97,7 +97,7 @@ STAGE PLANS: alias: src_thrift Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (lint is not null and (not mstringstring is null)) (type: boolean) + predicate: (lint is not null and mstringstring is not null) (type: boolean) Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: lint is not null (type: boolean), lintstring is not null (type: boolean), mstringstring is not null (type: boolean) diff --git a/ql/src/test/results/clientpositive/udf_size.q.out b/ql/src/test/results/clientpositive/udf_size.q.out index efb06a1..13594d2 100644 --- a/ql/src/test/results/clientpositive/udf_size.q.out +++ b/ql/src/test/results/clientpositive/udf_size.q.out @@ -38,7 +38,7 @@ STAGE PLANS: alias: src_thrift Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (lint is not null and (not mstringstring is null)) (type: boolean) + predicate: (lint is not null and mstringstring is not null) (type: boolean) Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int), -1 (type: int) diff --git a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out index e381757..91a585c 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out @@ -1191,15 +1191,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / key) (type: decimal(38,24)) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1270,15 +1270,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21)) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1339,15 +1339,15 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value <> 0) (type: boolean) - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value <> 0)) (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat