diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0abb788..d1ec723 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1892,8 +1892,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVEOPTGBYUSINGINDEX("hive.optimize.index.groupby", false, "Whether to enable optimization of group-by queries using Aggregate indexes."), - HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true, ""), - HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("none", "minimal", "more"), "Some select queries can be converted to single FETCH task minimizing latency.\n" + "Currently the query should be single sourced not having any subquery and should not have\n" + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 6ed5b13..602b4fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -191,8 +191,6 @@ ARCHIVE_ON_TABLE(10110, "ARCHIVE can only be run on partitions"), RESERVED_PART_VAL(10111, "Partition value contains a reserved substring"), OFFLINE_TABLE_OR_PARTITION(10113, "Query against an offline table or partition"), - OUTERJOIN_USES_FILTERS(10114, "The query results could be wrong. " + - "Turn on hive.outerjoin.supports.filters"), NEED_PARTITION_SPECIFICATION(10115, "Table is partitioned and partition specification is needed"), INVALID_METADATA(10116, "The metadata file could not be parsed "), NEED_TABLE_SPECIFICATION(10117, "Table name could be determined; It should be specified "), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ae1def1..66589fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2589,12 +2589,7 @@ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree, || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(0).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(0).add(joinCond); - } } else { /* * If the rhs references table sources and this QBJoinTree has a leftTree; @@ -2678,12 +2673,7 @@ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree, || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(1).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(1).add(joinCond); - } } else { joinTree.getFiltersForPushing().get(1).add(joinCond); } @@ -2698,24 +2688,14 @@ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree, } else if (rightCondAl1.size() != 0) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(0).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(0).add(joinCond); - } } else { joinTree.getFiltersForPushing().get(0).add(joinCond); } } else { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(1).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(1).add(joinCond); - } } else if (type.equals(JoinType.LEFTSEMI)) { joinTree.getExpressions().get(0).add(leftCondn); joinTree.getExpressions().get(1).add(rightCondn); @@ -2866,24 +2846,14 @@ private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, if (!leftAliasNull) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(0).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(0).add(joinCond); - } } else { joinTree.getFiltersForPushing().get(0).add(joinCond); } } else { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) { joinTree.getFilters().get(1).add(joinCond); - } else { - LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS.getErrorCodedMsg()); - joinTree.getFiltersForPushing().get(1).add(joinCond); - } } else { joinTree.getFiltersForPushing().get(1).add(joinCond); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index a872809..2e13dab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -53,7 +53,7 @@ private static final Logger LOG = LoggerFactory.getLogger(SetProcessor.class); private static final String prefix = "set: "; - private static final Set removedConfigs = Sets.newHashSet("hive.mapred.supports.subdirectories","hive.enforce.sorting","hive.enforce.bucketing"); + private static final Set removedConfigs = Sets.newHashSet("hive.mapred.supports.subdirectories","hive.enforce.sorting","hive.enforce.bucketing", "hive.outerjoin.supports.filters"); public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { diff --git a/ql/src/test/results/clientpositive/auto_join_filters.q.out b/ql/src/test/results/clientpositive/auto_join_filters.q.out index 2d4a043..e526284 100644 --- a/ql/src/test/results/clientpositive/auto_join_filters.q.out +++ b/ql/src/test/results/clientpositive/auto_join_filters.q.out @@ -310,7 +310,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -319,8 +319,8 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a L POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +4937935 +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -329,8 +329,8 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 -Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +3080335 +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -339,7 +339,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a F POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4939870 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -384,7 +384,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a L POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4937935 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -393,7 +393,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a L POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4937935 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -402,7 +402,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a L POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4937935 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -411,7 +411,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a L POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4937935 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -420,7 +420,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -429,7 +429,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -438,7 +438,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -447,7 +447,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -456,7 +456,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a F POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4939870 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -465,7 +465,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a F POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4939870 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -474,7 +474,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a F POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4939870 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -483,7 +483,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a F POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +4939870 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -501,7 +501,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -528,7 +528,7 @@ POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a R POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### -3078400 +3080335 PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1