diff --git a/build-common.xml b/build-common.xml index 581ded1..7fab5ff 100644 --- a/build-common.xml +++ b/build-common.xml @@ -57,7 +57,7 @@ - + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6c46a15..f4d0624 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3974,6 +3974,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (enforceBucketing || enforceSorting) { int maxReducers = conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS); + if (conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS) > 0) { + maxReducers = conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS); + } int numBuckets = dest_tab.getNumBuckets(); if (numBuckets > maxReducers) { multiFileSpray = true; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java new file mode 100644 index 0000000..340ce5b --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.hive.ql.MapRedStats; +import org.apache.hadoop.hive.ql.session.SessionState; + +/** + * + * VerifyNumReducersForBucketsHook. + * + * This hook is meant to be used with bucket_num_reducers.q . It checks whether the + * number of reducers has been correctly set. + */ +public class VerifyNumReducersForBucketsHook implements ExecuteWithHookContext { + + public void run(HookContext hookContext) { + SessionState ss = SessionState.get(); + Assert.assertNotNull("SessionState returned null"); + + List stats = ss.getLastMapRedStatsList(); + Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size()); + + Assert.assertEquals("NumReducers is incorrect", 10, stats.get(0).getNumReduce()); + } +} diff --git a/ql/src/test/queries/clientpositive/bucket_num_reducers.q b/ql/src/test/queries/clientpositive/bucket_num_reducers.q new file mode 100644 index 0000000..0b4b58f --- /dev/null +++ b/ql/src/test/queries/clientpositive/bucket_num_reducers.q @@ -0,0 +1,12 @@ +set hive.enforce.bucketing = true; +set hive.exec.mode.local.auto=false; +set mapred.reduce.tasks = 10; + +CREATE TABLE bucket_nr(key int, value string) CLUSTERED BY (key) INTO 50 BUCKETS; +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersForBucketsHook; + +insert overwrite table bucket_nr +select * from src; + +set hive.exec.post.hooks=; +drop table bucket_nr; diff --git a/ql/src/test/results/clientpositive/bucket_num_reducers.q.out b/ql/src/test/results/clientpositive/bucket_num_reducers.q.out new file mode 100644 index 0000000..591191c --- /dev/null +++ b/ql/src/test/results/clientpositive/bucket_num_reducers.q.out @@ -0,0 +1,14 @@ +PREHOOK: query: CREATE TABLE bucket_nr(key int, value string) CLUSTERED BY (key) INTO 50 BUCKETS +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE bucket_nr(key int, value string) CLUSTERED BY (key) INTO 50 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@bucket_nr +PREHOOK: query: insert overwrite table bucket_nr +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket_nr +PREHOOK: query: drop table bucket_nr +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@bucket_nr +PREHOOK: Output: default@bucket_nr