commit ac717d5283dae2c16a583bd817ee5f0323c622bf Author: Sahil Takiar Date: Thu Jan 18 12:35:55 2018 -0800 HIVE-18485: Add more unit tests for hive.strict.checks.* properties diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 05c2acd0ff..2ef4b575d9 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4958,7 +4958,7 @@ public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { private static final String NO_LIMIT_MSG = makeMessage( "Order by-s without limit", ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY); - private static final String NO_PARTITIONLESS_MSG = makeMessage( + public static final String NO_PARTITIONLESS_MSG = makeMessage( "Queries against partitioned tables without a partition filter", ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY); private static final String NO_COMPARES_MSG = makeMessage( @@ -4969,7 +4969,7 @@ public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { "Load into bucketed tables", ConfVars.HIVE_STRICT_CHECKS_BUCKETING); private static String makeMessage(String what, ConfVars setting) { - return what + " are disabled for safety reasons. If you know what you are doing, please set" + return what + " are disabled for safety reasons. If you know what you are doing, please set " + setting.varname + " to false and that " + ConfVars.HIVEMAPREDMODE.varname + " is not" + " set to 'strict' to proceed. Note that if you may get errors or incorrect results if" + " you make a mistake while using some of the unsafe features."; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 5baac1894d..134faeeec9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -114,15 +114,12 @@ CLUSTERBY_ORDERBY_CONFLICT(10050, "Cannot have both CLUSTER BY and ORDER BY clauses"), NO_LIMIT_WITH_ORDERBY(10051, "In strict mode, if ORDER BY is specified, " + "LIMIT must also be specified"), - NO_CARTESIAN_PRODUCT(10052, "In strict mode, cartesian product is not allowed. " - + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"), UNION_NOTIN_SUBQ(10053, "Top level UNION is not supported currently; " + "use a subquery for the UNION"), INVALID_INPUT_FORMAT_TYPE(10054, "Input format must implement InputFormat"), INVALID_OUTPUT_FORMAT_TYPE(10055, "Output Format must implement HiveOutputFormat, " + "otherwise it should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat"), - NO_VALID_PARTN(10056, "The query does not reference any valid partition. " - + "To run this query, set hive.mapred.mode=nonstrict"), + NO_VALID_PARTN(10056, HiveConf.StrictChecks.NO_PARTITIONLESS_MSG), NO_OUTER_MAPJOIN(10057, "MAPJOIN cannot be performed with OUTER JOIN"), INVALID_MAPJOIN_HINT(10058, "All tables are specified as map-table for join"), INVALID_MAPJOIN_TABLE(10059, "Result of a union cannot be a map table"), @@ -206,12 +203,6 @@ INCOMPATIBLE_SCHEMA(10120, "The existing table is not compatible with the import spec. "), EXIM_FOR_NON_NATIVE(10121, "Export/Import cannot be done for a non-native table. "), INSERT_INTO_BUCKETIZED_TABLE(10122, "Bucketized tables do not support INSERT INTO:"), - NO_COMPARE_BIGINT_STRING(10123, "In strict mode, comparing bigints and strings is not allowed, " - + "it may result in a loss of precision. " - + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"), - NO_COMPARE_BIGINT_DOUBLE(10124, "In strict mode, comparing bigints and doubles is not allowed, " - + "it may result in a loss of precision. " - + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"), PARTSPEC_DIFFER_FROM_SCHEMA(10125, "Partition columns in partition specification are " + "not the same as that defined in the table schema. " + "The names and orders have to be exactly the same."), diff --git a/ql/src/test/queries/clientnegative/alter_view_failure6_2.q b/ql/src/test/queries/clientnegative/alter_view_failure6_2.q new file mode 100644 index 0000000000..aea26241f2 --- /dev/null +++ b/ql/src/test/queries/clientnegative/alter_view_failure6_2.q @@ -0,0 +1,14 @@ +set hive.strict.checks.bucketing=false; + +DROP VIEW xxx7; +CREATE VIEW xxx7 +PARTITIONED ON (key) +AS +SELECT hr,key FROM srcpart; + +RESET hive.mapred.mode; +SET hive.strict.checks.large.query=true; + +-- strict mode should cause this to fail since view partition +-- predicate does not correspond to an underlying table partition predicate +ALTER VIEW xxx7 ADD PARTITION (key=10); diff --git a/ql/src/test/queries/clientnegative/compare_double_bigint_2.q b/ql/src/test/queries/clientnegative/compare_double_bigint_2.q new file mode 100644 index 0000000000..bc2d7cd157 --- /dev/null +++ b/ql/src/test/queries/clientnegative/compare_double_bigint_2.q @@ -0,0 +1,8 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.type.safety=true; + +-- This should fail until we fix the issue with precision when casting a bigint to a double + +select * from src where cast(1 as bigint) = cast(1.0 as double) limit 10; diff --git a/ql/src/test/queries/clientnegative/compare_string_bigint_2.q b/ql/src/test/queries/clientnegative/compare_string_bigint_2.q new file mode 100644 index 0000000000..a439e33a3c --- /dev/null +++ b/ql/src/test/queries/clientnegative/compare_string_bigint_2.q @@ -0,0 +1,8 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.type.safety=true; + +--This should fail until we fix the issue with precision when casting a bigint to a double + +select * from src where cast(1 as bigint) = '1' limit 10; diff --git a/ql/src/test/queries/clientnegative/input4_2.q b/ql/src/test/queries/clientnegative/input4_2.q new file mode 100644 index 0000000000..ca8bcda6d7 --- /dev/null +++ b/ql/src/test/queries/clientnegative/input4_2.q @@ -0,0 +1,8 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.cartesian.product=true; + +select * from srcpart a join + (select b.key, count(1) as count from srcpart b where b.ds = '2008-04-08' and b.hr = '14' group by b.key) subq + where a.ds = '2008-04-08' and a.hr = '11' limit 10; diff --git a/ql/src/test/queries/clientnegative/input_part0_neg_2.q b/ql/src/test/queries/clientnegative/input_part0_neg_2.q new file mode 100644 index 0000000000..47348de751 --- /dev/null +++ b/ql/src/test/queries/clientnegative/input_part0_neg_2.q @@ -0,0 +1,6 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.large.query=true; + +SELECT x.* FROM SRCPART x WHERE key = '2008-04-08'; diff --git a/ql/src/test/queries/clientnegative/strict_join.q b/ql/src/test/queries/clientnegative/strict_join.q index 48b4a60a15..66b5963191 100644 --- a/ql/src/test/queries/clientnegative/strict_join.q +++ b/ql/src/test/queries/clientnegative/strict_join.q @@ -3,3 +3,8 @@ set hive.strict.checks.bucketing=false; set hive.mapred.mode=strict; SELECT * FROM src src1 JOIN src src2; + +reset hive.mapred.mode; +set hive.strict.checks.cartesian.product=true; + +SELECT * FROM src src1 JOIN src src2; diff --git a/ql/src/test/queries/clientnegative/strict_join_2.q b/ql/src/test/queries/clientnegative/strict_join_2.q new file mode 100644 index 0000000000..dc5f87b733 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_join_2.q @@ -0,0 +1,6 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.cartesian.product=true; + +SELECT * FROM src src1 JOIN src src2; diff --git a/ql/src/test/queries/clientnegative/strict_pruning_2.q b/ql/src/test/queries/clientnegative/strict_pruning_2.q new file mode 100644 index 0000000000..393ef74126 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_pruning_2.q @@ -0,0 +1,9 @@ +set hive.strict.checks.bucketing=false; + +reset hive.mapred.mode; +set hive.strict.checks.large.query=true; + +EXPLAIN +SELECT count(1) FROM srcPART; + +SELECT count(1) FROM srcPART; diff --git a/ql/src/test/results/clientnegative/alter_view_failure6.q.out b/ql/src/test/results/clientnegative/alter_view_failure6.q.out index 827dd938bc..0227ba7da2 100644 --- a/ql/src/test/results/clientnegative/alter_view_failure6.q.out +++ b/ql/src/test/results/clientnegative/alter_view_failure6.q.out @@ -19,5 +19,5 @@ POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@xxx7 POSTHOOK: Lineage: xxx7.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -FAILED: SemanticException Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" -FAILED: SemanticException [Error 10056]: The query does not reference any valid partition. To run this query, set hive.mapred.mode=nonstrict +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out b/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out new file mode 100644 index 0000000000..0227ba7da2 --- /dev/null +++ b/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out @@ -0,0 +1,23 @@ +PREHOOK: query: DROP VIEW xxx7 +PREHOOK: type: DROPVIEW +POSTHOOK: query: DROP VIEW xxx7 +POSTHOOK: type: DROPVIEW +PREHOOK: query: CREATE VIEW xxx7 +PARTITIONED ON (key) +AS +SELECT hr,key FROM srcpart +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@srcpart +PREHOOK: Output: database:default +PREHOOK: Output: default@xxx7 +POSTHOOK: query: CREATE VIEW xxx7 +PARTITIONED ON (key) +AS +SELECT hr,key FROM srcpart +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@srcpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@xxx7 +POSTHOOK: Lineage: xxx7.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_double_bigint.q.out b/ql/src/test/results/clientnegative/compare_double_bigint.q.out index 26a64fc8f2..39d17172d8 100644 --- a/ql/src/test/results/clientnegative/compare_double_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_double_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out b/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out new file mode 100644 index 0000000000..39d17172d8 --- /dev/null +++ b/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_string_bigint.q.out b/ql/src/test/results/clientnegative/compare_string_bigint.q.out index d80b3141ad..32bfef49c4 100644 --- a/ql/src/test/results/clientnegative/compare_string_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_string_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out b/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out new file mode 100644 index 0000000000..32bfef49c4 --- /dev/null +++ b/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input4.q.out b/ql/src/test/results/clientnegative/input4.q.out index 0e047104f2..776a1ec82c 100644 --- a/ql/src/test/results/clientnegative/input4.q.out +++ b/ql/src/test/results/clientnegative/input4.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input4_2.q.out b/ql/src/test/results/clientnegative/input4_2.q.out new file mode 100644 index 0000000000..776a1ec82c --- /dev/null +++ b/ql/src/test/results/clientnegative/input4_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input_part0_neg.q.out b/ql/src/test/results/clientnegative/input_part0_neg.q.out index 46a0f6139a..1e812d8cf3 100644 --- a/ql/src/test/results/clientnegative/input_part0_neg.q.out +++ b/ql/src/test/results/clientnegative/input_part0_neg.q.out @@ -1 +1 @@ -FAILED: SemanticException Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/input_part0_neg_2.q.out b/ql/src/test/results/clientnegative/input_part0_neg_2.q.out new file mode 100644 index 0000000000..1e812d8cf3 --- /dev/null +++ b/ql/src/test/results/clientnegative/input_part0_neg_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/strict_join.q.out b/ql/src/test/results/clientnegative/strict_join.q.out index 0e047104f2..776a1ec82c 100644 --- a/ql/src/test/results/clientnegative/strict_join.q.out +++ b/ql/src/test/results/clientnegative/strict_join.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/strict_join_2.q.out b/ql/src/test/results/clientnegative/strict_join_2.q.out new file mode 100644 index 0000000000..776a1ec82c --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_join_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/strict_orderby.q.out b/ql/src/test/results/clientnegative/strict_orderby.q.out index 3cc2a0c4b0..a4f50d1827 100644 --- a/ql/src/test/results/clientnegative/strict_orderby.q.out +++ b/ql/src/test/results/clientnegative/strict_orderby.q.out @@ -1 +1 @@ -FAILED: SemanticException 4:47 Order by-s without limit are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features.. Error encountered near token 'key' +FAILED: SemanticException 4:47 Order by-s without limit are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features.. Error encountered near token 'key' diff --git a/ql/src/test/results/clientnegative/strict_pruning.q.out b/ql/src/test/results/clientnegative/strict_pruning.q.out index 76b9083b71..4fad801bf8 100644 --- a/ql/src/test/results/clientnegative/strict_pruning.q.out +++ b/ql/src/test/results/clientnegative/strict_pruning.q.out @@ -1 +1 @@ -FAILED: SemanticException Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please sethive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/strict_pruning_2.q.out b/ql/src/test/results/clientnegative/strict_pruning_2.q.out new file mode 100644 index 0000000000..4fad801bf8 --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_pruning_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart"