commit 45a75686616b368b870448e58fe4ff9fa90ee956 Author: Sahil Takiar Date: Mon Jan 29 17:58:21 2018 -0800 HIVE-18552: Split hive.strict.checks.large.query into two configs diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b7d3e99e1a..0b02409575 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1090,11 +1090,14 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.txn.valid.txns,hive.script.operator.env.blacklist", "Comma separated list of keys from the configuration file not to convert to environment " + "variables when invoking the script operator"), - HIVE_STRICT_CHECKS_LARGE_QUERY("hive.strict.checks.large.query", false, + HIVE_STRICT_CHECKS_ORDERBY_NO_LIMIT("hive.strict.checks.orderby.no.limit", false, "Enabling strict large query checks disallows the following:\n" + " Orderby without limit.\n" + + "Note that this check currently does not consider data size, only the query pattern."), + HIVE_STRICT_CHECKS_NO_PARTITION_FILTER("hive.strict.checks.no.partition.filter", false, + "Enabling strict large query checks disallows the following:\n" + " No partition being picked up for a query against partitioned table.\n" + - "Note that these checks currently do not consider data size, only the query pattern."), + "Note that this check currently does not consider data size, only the query pattern."), HIVE_STRICT_CHECKS_TYPE_SAFETY("hive.strict.checks.type.safety", true, "Enabling strict type safety checks disallows the following:\n" + " Comparing bigints and strings.\n" + @@ -4965,10 +4968,10 @@ public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { public static class StrictChecks { private static final String NO_LIMIT_MSG = makeMessage( - "Order by-s without limit", ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY); + "Order by-s without limit", ConfVars.HIVE_STRICT_CHECKS_ORDERBY_NO_LIMIT); public static final String NO_PARTITIONLESS_MSG = makeMessage( "Queries against partitioned tables without a partition filter", - ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY); + ConfVars.HIVE_STRICT_CHECKS_NO_PARTITION_FILTER); private static final String NO_COMPARES_MSG = makeMessage( "Unsafe compares between different types", ConfVars.HIVE_STRICT_CHECKS_TYPE_SAFETY); private static final String NO_CARTESIAN_MSG = makeMessage( @@ -4978,17 +4981,17 @@ public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { private static String makeMessage(String what, ConfVars setting) { return what + " are disabled for safety reasons. If you know what you are doing, please set " - + setting.varname + " to false and that " + ConfVars.HIVEMAPREDMODE.varname + " is not" - + " set to 'strict' to proceed. Note that if you may get errors or incorrect results if" - + " you make a mistake while using some of the unsafe features."; + + setting.varname + " to false and make sure that " + ConfVars.HIVEMAPREDMODE.varname + + " is not set to 'strict' to proceed. Note that you may get errors or incorrect " + + "results if you make a mistake while using some of the unsafe features."; } public static String checkNoLimit(Configuration conf) { - return isAllowed(conf, ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY) ? null : NO_LIMIT_MSG; + return isAllowed(conf, ConfVars.HIVE_STRICT_CHECKS_ORDERBY_NO_LIMIT) ? null : NO_LIMIT_MSG; } public static String checkNoPartitionFilter(Configuration conf) { - return isAllowed(conf, ConfVars.HIVE_STRICT_CHECKS_LARGE_QUERY) + return isAllowed(conf, ConfVars.HIVE_STRICT_CHECKS_NO_PARTITION_FILTER) ? null : NO_PARTITIONLESS_MSG; } diff --git a/ql/src/test/queries/clientnegative/alter_view_failure6_2.q b/ql/src/test/queries/clientnegative/alter_view_failure6_2.q index aea26241f2..730a8f5ac3 100644 --- a/ql/src/test/queries/clientnegative/alter_view_failure6_2.q +++ b/ql/src/test/queries/clientnegative/alter_view_failure6_2.q @@ -7,7 +7,7 @@ AS SELECT hr,key FROM srcpart; RESET hive.mapred.mode; -SET hive.strict.checks.large.query=true; +SET hive.strict.checks.no.partition.filter=true; -- strict mode should cause this to fail since view partition -- predicate does not correspond to an underlying table partition predicate diff --git a/ql/src/test/queries/clientnegative/input_part0_neg_2.q b/ql/src/test/queries/clientnegative/input_part0_neg_2.q index 47348de751..91707448e2 100644 --- a/ql/src/test/queries/clientnegative/input_part0_neg_2.q +++ b/ql/src/test/queries/clientnegative/input_part0_neg_2.q @@ -1,6 +1,6 @@ set hive.strict.checks.bucketing=false; reset hive.mapred.mode; -set hive.strict.checks.large.query=true; +set hive.strict.checks.no.partition.filter=true; SELECT x.* FROM SRCPART x WHERE key = '2008-04-08'; diff --git a/ql/src/test/queries/clientnegative/strict_orderby_2.q b/ql/src/test/queries/clientnegative/strict_orderby_2.q new file mode 100644 index 0000000000..0a93af722d --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_orderby_2.q @@ -0,0 +1,8 @@ +reset hive.mapred.mode; +set hive.strict.checks.orderby.no.limit=true; + +EXPLAIN +SELECT src.key, src.value from src order by src.key; + +SELECT src.key, src.value from src order by src.key; + diff --git a/ql/src/test/queries/clientnegative/strict_pruning_2.q b/ql/src/test/queries/clientnegative/strict_pruning_2.q index 393ef74126..718566d4c9 100644 --- a/ql/src/test/queries/clientnegative/strict_pruning_2.q +++ b/ql/src/test/queries/clientnegative/strict_pruning_2.q @@ -1,7 +1,5 @@ -set hive.strict.checks.bucketing=false; - reset hive.mapred.mode; -set hive.strict.checks.large.query=true; +set hive.strict.checks.no.partition.filter=true; EXPLAIN SELECT count(1) FROM srcPART; diff --git a/ql/src/test/results/clientnegative/alter_view_failure6.q.out b/ql/src/test/results/clientnegative/alter_view_failure6.q.out index 0227ba7da2..f295543548 100644 --- a/ql/src/test/results/clientnegative/alter_view_failure6.q.out +++ b/ql/src/test/results/clientnegative/alter_view_failure6.q.out @@ -19,5 +19,5 @@ POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@xxx7 POSTHOOK: Lineage: xxx7.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out b/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out index 0227ba7da2..f295543548 100644 --- a/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out +++ b/ql/src/test/results/clientnegative/alter_view_failure6_2.q.out @@ -19,5 +19,5 @@ POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@xxx7 POSTHOOK: Lineage: xxx7.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_double_bigint.q.out b/ql/src/test/results/clientnegative/compare_double_bigint.q.out index 39d17172d8..a577f0e92d 100644 --- a/ql/src/test/results/clientnegative/compare_double_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_double_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out b/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out index 39d17172d8..a577f0e92d 100644 --- a/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out +++ b/ql/src/test/results/clientnegative/compare_double_bigint_2.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments '1.0': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_string_bigint.q.out b/ql/src/test/results/clientnegative/compare_string_bigint.q.out index 32bfef49c4..01ea35cd21 100644 --- a/ql/src/test/results/clientnegative/compare_string_bigint.q.out +++ b/ql/src/test/results/clientnegative/compare_string_bigint.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out b/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out index 32bfef49c4..01ea35cd21 100644 --- a/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out +++ b/ql/src/test/results/clientnegative/compare_string_bigint_2.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares between different types are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.type.safety to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input4.q.out b/ql/src/test/results/clientnegative/input4.q.out index 776a1ec82c..c857f92e67 100644 --- a/ql/src/test/results/clientnegative/input4.q.out +++ b/ql/src/test/results/clientnegative/input4.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input4_2.q.out b/ql/src/test/results/clientnegative/input4_2.q.out index 776a1ec82c..c857f92e67 100644 --- a/ql/src/test/results/clientnegative/input4_2.q.out +++ b/ql/src/test/results/clientnegative/input4_2.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/input_part0_neg.q.out b/ql/src/test/results/clientnegative/input_part0_neg.q.out index 1e812d8cf3..85c16b9837 100644 --- a/ql/src/test/results/clientnegative/input_part0_neg.q.out +++ b/ql/src/test/results/clientnegative/input_part0_neg.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/input_part0_neg_2.q.out b/ql/src/test/results/clientnegative/input_part0_neg_2.q.out index 1e812d8cf3..85c16b9837 100644 --- a/ql/src/test/results/clientnegative/input_part0_neg_2.q.out +++ b/ql/src/test/results/clientnegative/input_part0_neg_2.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "default.srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/strict_join.q.out b/ql/src/test/results/clientnegative/strict_join.q.out index 776a1ec82c..c857f92e67 100644 --- a/ql/src/test/results/clientnegative/strict_join.q.out +++ b/ql/src/test/results/clientnegative/strict_join.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/strict_join_2.q.out b/ql/src/test/results/clientnegative/strict_join_2.q.out index 776a1ec82c..c857f92e67 100644 --- a/ql/src/test/results/clientnegative/strict_join_2.q.out +++ b/ql/src/test/results/clientnegative/strict_join_2.q.out @@ -1 +1 @@ -FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. +FAILED: SemanticException Cartesian products are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.cartesian.product to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. diff --git a/ql/src/test/results/clientnegative/strict_orderby.q.out b/ql/src/test/results/clientnegative/strict_orderby.q.out index a4f50d1827..3f7ae26ab1 100644 --- a/ql/src/test/results/clientnegative/strict_orderby.q.out +++ b/ql/src/test/results/clientnegative/strict_orderby.q.out @@ -1 +1 @@ -FAILED: SemanticException 4:47 Order by-s without limit are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features.. Error encountered near token 'key' +FAILED: SemanticException 4:47 Order by-s without limit are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.orderby.no.limit to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features.. Error encountered near token 'key' diff --git a/ql/src/test/results/clientnegative/strict_orderby_2.q.out b/ql/src/test/results/clientnegative/strict_orderby_2.q.out new file mode 100644 index 0000000000..3f7ae26ab1 --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_orderby_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException 4:47 Order by-s without limit are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.orderby.no.limit to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features.. Error encountered near token 'key' diff --git a/ql/src/test/results/clientnegative/strict_pruning.q.out b/ql/src/test/results/clientnegative/strict_pruning.q.out index 4fad801bf8..b50d411cb8 100644 --- a/ql/src/test/results/clientnegative/strict_pruning.q.out +++ b/ql/src/test/results/clientnegative/strict_pruning.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart" diff --git a/ql/src/test/results/clientnegative/strict_pruning_2.q.out b/ql/src/test/results/clientnegative/strict_pruning_2.q.out index 4fad801bf8..b50d411cb8 100644 --- a/ql/src/test/results/clientnegative/strict_pruning_2.q.out +++ b/ql/src/test/results/clientnegative/strict_pruning_2.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.large.query to false and that hive.mapred.mode is not set to 'strict' to proceed. Note that if you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart" +FAILED: SemanticException [Error 10056]: Queries against partitioned tables without a partition filter are disabled for safety reasons. If you know what you are doing, please set hive.strict.checks.no.partition.filter to false and make sure that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get errors or incorrect results if you make a mistake while using some of the unsafe features. No partition predicate for Alias "srcpart" Table "srcpart"