diff --git shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index 4f5d4fa..082cedd 100644 --- shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -40,6 +40,7 @@ import javax.security.auth.Subject; import javax.security.auth.login.LoginException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; @@ -510,15 +511,28 @@ public void abortTask(TaskAttemptContext taskContext) { } } public void prepareJobOutput(JobConf conf) { - conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class); + String outCommit = conf.get("mapred.output.committer.class"); - // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) - // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); + if(outCommit!=null && StringUtils.isNotBlank(outCommit)) + { + try { + conf.setOutputCommitter((Class) (Class.forName(outCommit))); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e.getMessage()); + } + } + else + { + conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class); - // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) - // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); + // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) + // but can be backported. So we disable setup/cleanup in all versions >= 0.19 + conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); + + // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) + // but can be backported. So we disable setup/cleanup in all versions >= 0.19 + conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); + } } @Override diff --git shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index e205caa..fc724e01 100644 --- shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -423,15 +424,28 @@ public void abortTask(TaskAttemptContext taskContext) { } } public void prepareJobOutput(JobConf conf) { - conf.setOutputCommitter(NullOutputCommitter.class); + String outCommit = conf.get("mapred.output.committer.class"); + + if(outCommit!=null && StringUtils.isNotBlank(outCommit)) + { + try { + conf.setOutputCommitter((Class) (Class.forName(outCommit))); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e.getMessage()); + } + } + else + { + conf.setOutputCommitter(NullOutputCommitter.class); - // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) - // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); + // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) + // but can be backported. So we disable setup/cleanup in all versions >= 0.19 + conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); - // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) - // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); + // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) + // but can be backported. So we disable setup/cleanup in all versions >= 0.19 + conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); + } } @Override