Index: build-common.xml
===================================================================
--- build-common.xml (revision 1433681)
+++ build-common.xml (working copy)
@@ -57,7 +57,7 @@
-
+
Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1433681)
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy)
@@ -4544,7 +4544,7 @@
}
private int getReducersBucketing(int totalFiles, int maxReducers) {
- int numFiles = totalFiles / maxReducers;
+ int numFiles = (int)Math.ceil((double)totalFiles / (double)maxReducers);
while (true) {
if (totalFiles % numFiles == 0) {
return totalFiles / numFiles;
Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java
===================================================================
--- ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java (revision 1433681)
+++ ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersForBucketsHook.java (working copy)
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.hooks;
-
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hive.ql.MapRedStats;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-/**
- *
- * VerifyNumReducersForBucketsHook.
- *
- * This hook is meant to be used with bucket_num_reducers.q . It checks whether the
- * number of reducers has been correctly set.
- */
-public class VerifyNumReducersForBucketsHook implements ExecuteWithHookContext {
-
- public void run(HookContext hookContext) {
- SessionState ss = SessionState.get();
- Assert.assertNotNull("SessionState returned null");
-
- List stats = ss.getLastMapRedStatsList();
- Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size());
-
- Assert.assertEquals("NumReducers is incorrect", 10, stats.get(0).getNumReduce());
- }
-}
Index: ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java
===================================================================
--- ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java (revision 0)
+++ ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyNumReducersHook.java (working copy)
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.ql.MapRedStats;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ *
+ * VerifyNumReducersHook.
+ *
+ * Provided a query involves exactly 1 map reduce job, this hook can be used to verify that the
+ * number of reducers matches what is expected.
+ *
+ * Use the config VerifyNumReducersHook.num.reducers to specify the expected number of reducers.
+ */
+public class VerifyNumReducersHook implements ExecuteWithHookContext {
+
+ public static final String BUCKET_CONFIG = "VerifyNumReducersHook.num.reducers";
+
+ public void run(HookContext hookContext) {
+ SessionState ss = SessionState.get();
+ Assert.assertNotNull("SessionState returned null");
+
+ int expectedReducers = hookContext.getConf().getInt(BUCKET_CONFIG, 0);
+ List stats = ss.getLastMapRedStatsList();
+ Assert.assertEquals("Number of MapReduce jobs is incorrect", 1, stats.size());
+
+ Assert.assertEquals("NumReducers is incorrect", expectedReducers, stats.get(0).getNumReduce());
+ }
+}
Index: ql/src/test/queries/clientpositive/bucket_num_reducers.q
===================================================================
--- ql/src/test/queries/clientpositive/bucket_num_reducers.q (revision 1433681)
+++ ql/src/test/queries/clientpositive/bucket_num_reducers.q (working copy)
@@ -6,7 +6,8 @@
-- and uses a post-hook to confirm that 10 tasks were created
CREATE TABLE bucket_nr(key int, value string) CLUSTERED BY (key) INTO 50 BUCKETS;
-set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersForBucketsHook;
+set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook;
+set VerifyNumReducersHook.num.reducers=10;
insert overwrite table bucket_nr
select * from src;
Index: ql/src/test/queries/clientpositive/bucket_num_reducers2.q
===================================================================
--- ql/src/test/queries/clientpositive/bucket_num_reducers2.q (revision 0)
+++ ql/src/test/queries/clientpositive/bucket_num_reducers2.q (working copy)
@@ -0,0 +1,13 @@
+set hive.enforce.bucketing = true;
+set hive.exec.mode.local.auto=false;
+set hive.exec.reducers.max = 2;
+
+-- This test sets the maximum number of reduce tasks to 2 for overwriting a
+-- table with 3 buckets, and uses a post-hook to confirm that 1 reducer was used
+
+CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS;
+set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyNumReducersHook;
+set VerifyNumReducersHook.num.reducers=1;
+
+insert overwrite table test_table
+select * from src;
Index: ql/src/test/results/clientpositive/bucket_num_reducers2.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucket_num_reducers2.q.out (revision 0)
+++ ql/src/test/results/clientpositive/bucket_num_reducers2.q.out (working copy)
@@ -0,0 +1,16 @@
+PREHOOK: query: -- This test sets the maximum number of reduce tasks to 2 for overwriting a
+-- table with 3 buckets, and uses a post-hook to confirm that 1 reducer was used
+
+CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- This test sets the maximum number of reduce tasks to 2 for overwriting a
+-- table with 3 buckets, and uses a post-hook to confirm that 2 reducers were used
+
+CREATE TABLE test_table(key int, value string) CLUSTERED BY (key) INTO 3 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_table
+PREHOOK: query: insert overwrite table test_table
+select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_table