Index: build-common.xml =================================================================== --- build-common.xml (revision 1433681) +++ build-common.xml (working copy) @@ -57,7 +57,7 @@ - + Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1433681) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -4544,7 +4544,7 @@ } private int getReducersBucketing(int totalFiles, int maxReducers) { - int numFiles = totalFiles / maxReducers; + int numFiles = (int)Math.ceil((double)totalFiles / (double)maxReducers); while (true) { if (totalFiles % numFiles == 0) { return totalFiles / numFiles; Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java (revision 1433681) +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java (working copy) @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.hooks; - -import java.util.List; - -import junit.framework.Assert; - -import org.apache.hadoop.hive.ql.MapRedStats; -import org.apache.hadoop.hive.ql.session.SessionState; - -/** - * - * VerifyNumReducersForBucketsHook. - * - * This hook is meant to be used with bucket_num_reducers.q . It checks whether the - * number of reducers has been correctly set. - */ -public class VerifyNumReducersForBucketsHook implements ExecuteWithHookContext { - - public void run(HookContext hookContext) { - SessionState ss = SessionState.get(); - Assert.assertNotNull("SessionState returned null"); - - List stats = ss.getLastMapRedStatsList(); - Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size()); - - Assert.assertEquals("NumReducers is incorrect", 10, stats.get(0).getNumReduce()); - } -} Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java (working copy) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.hive.ql.MapRedStats; +import org.apache.hadoop.hive.ql.session.SessionState; + +/** + * + * VerifyNumReducersHook. + * + * Provided a query involves exactly 1 map reduce job, this hook can be used to verify that the + * number of reducers matches what is expected. + * + * Use the config VerifyNumReducersHook.num.reducers to specify the expected number of reducers. + */ +public class VerifyNumReducersHook implements ExecuteWithHookContext { + + public static final String BUCKET_CONFIG = "VerifyNumReducersHook.num.reducers"; + + public void run(HookContext hookContext) { + SessionState ss = SessionState.get(); + Assert.assertNotNull("SessionState returned null"); + + int expectedReducers = hookContext.getConf().getInt(BUCKET_CONFIG, 0); + List stats = ss.getLastMapRedStatsList(); + Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size()); + + Assert.assertEquals("NumReducers is incorrect", expectedReducers, stats.get(0).getNumReduce()); + } +} Index: ql/src/test/queries/clientpositive/bucket_num_reducers.q =================================================================== --- ql/src/test/queries/clientpositive/bucket_num_reducers.q (revision 1433681) +++ ql/src/test/queries/clientpositive/bucket_num_reducers.q (working copy) @@ -6,7 +6,8 @@ -- and uses a post-hook to confirm that 10 tasks were created CREATE TABLE bucket_nr(key int, value string) CLUSTERED BY (key) INTO 50 BUCKETS; -set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersForBucketsHook; +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook; +set VerifyNumReducersHook.num.reducers=10; insert overwrite table bucket_nr select * from src; Index: ql/src/test/queries/clientpositive/bucket_num_reducers2.q =================================================================== --- ql/src/test/queries/clientpositive/bucket_num_reducers2.q (revision 0) +++ ql/src/test/queries/clientpositive/bucket_num_reducers2.q (working copy) @@ -0,0 +1,13 @@ +set hive.enforce.bucketing = true; +set hive.exec.mode.local.auto=false; +set hive.exec.reducers.max = 2; + +-- This test sets the maximum number of reduce tasks to 2 for overwriting a +-- table with 3 buckets, and uses a post-hook to confirm that 1 reducer was used + +CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS; +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook; +set VerifyNumReducersHook.num.reducers=1; + +insert overwrite table test_table +select * from src; Index: ql/src/test/results/clientpositive/bucket_num_reducers2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucket_num_reducers2.q.out (revision 0) +++ ql/src/test/results/clientpositive/bucket_num_reducers2.q.out (working copy) @@ -0,0 +1,16 @@ +PREHOOK: query: -- This test sets the maximum number of reduce tasks to 2 for overwriting a +-- table with 3 buckets, and uses a post-hook to confirm that 1 reducer was used + +CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- This test sets the maximum number of reduce tasks to 2 for overwriting a +-- table with 3 buckets, and uses a post-hook to confirm that 2 reducers were used + +CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test_table +PREHOOK: query: insert overwrite table test_table +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_table