diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index dcfe29a..86a1721 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -191,7 +191,7 @@ COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", ""), COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", ""), BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (1000 * 1000 * 1000)), - MAXREDUCERS("hive.exec.reducers.max", 999), + MAXREDUCERS("hive.exec.reducers.max", 997), PREEXECHOOKS("hive.exec.pre.hooks", ""), POSTEXECHOOKS("hive.exec.post.hooks", ""), ONFAILUREHOOKS("hive.exec.failure.hooks", ""), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 7250432..75dfbd1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -203,6 +203,20 @@ public static final String MAPRED_MAPPER_CLASS = "mapred.mapper.class"; public static final String MAPRED_REDUCER_CLASS = "mapred.reducer.class"; + public static final int[] PRIMES = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, + 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, + 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, + 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, + 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, + 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, + 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, + 523, 541, 547, 557, 563, 569, 571, 577, 587, 593, 599, 601, 607, 613, + 617, 619, 631, 641, 643, 647, 653, 659, 661, 673, 677, 683, 691, 701, + 709, 719, 727, 733, 739, 743, 751, 757, 761, 769, 773, 787, 797, 809, + 811, 821, 823, 827, 829, 839, 853, 857, 859, 863, 877, 881, 883, 887, + 907, 911, 919, 929, 937, 941, 947, 953, 967, 971, 977, 983, 991, 997, + 1009, 1013, 1019, 1021 }; + /** * ReduceField: * KEY: record key @@ -2912,7 +2926,23 @@ public static int estimateReducers(long totalInputFileSize, long bytesPerReducer // Otherwise use the smallest power of two greater than the original number of reducers reducers = reducersPowerTwo; } + } else if(PRIMES[PRIMES.length-1] > reducers) { + int match = Arrays.binarySearch(PRIMES, reducers); + /* if the match is >= 0, then it is already in the list. */ + if(match < 0) { + int off = -1*match; + /* -1*n is >= 1 */ + int morePrime = PRIMES[off]; + int lessPrime = PRIMES[off-1]; + if(morePrime < maxReducers) { + reducers = morePrime; + } else { + /* invariant: lessPrime < reducers <= maxReducers */ + reducers = lessPrime; + } } + } + return reducers; }