commit c9cbb9ad2e0f010dd423ccf740c21d6dbcd85720 Author: Alan Gates Date: Fri Nov 21 16:03:31 2014 -0800 HIVE-8944 Fix shimming issue in CompactorMR diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 1d3cf00..3cdbc6c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; @@ -697,9 +698,10 @@ public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException @Override public void commitJob(JobContext context) throws IOException { - Path tmpLocation = new Path(context.getJobConf().get(TMP_LOCATION)); - Path finalLocation = new Path(context.getJobConf().get(FINAL_LOCATION)); - FileSystem fs = tmpLocation.getFileSystem(context.getJobConf()); + JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); + Path tmpLocation = new Path(conf.get(TMP_LOCATION)); + Path finalLocation = new Path(conf.get(FINAL_LOCATION)); + FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Moving contents of " + tmpLocation.toString() + " to " + finalLocation.toString()); @@ -713,8 +715,9 @@ public void commitJob(JobContext context) throws IOException { @Override public void abortJob(JobContext context, int status) throws IOException { - Path tmpLocation = new Path(context.getJobConf().get(TMP_LOCATION)); - FileSystem fs = tmpLocation.getFileSystem(context.getJobConf()); + JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); + Path tmpLocation = new Path(conf.get(TMP_LOCATION)); + FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Removing " + tmpLocation.toString()); fs.delete(tmpLocation, true); } diff --git shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index 76df71c..a524573 100644 --- shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -889,6 +889,11 @@ public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext con } @Override + public JobConf getJobConf(JobContext context) { + return context.getJobConf(); + } + + @Override public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { boolean origDisableHDFSCache = conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false); diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 127f6e6..1592c32 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -512,6 +512,11 @@ public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext con } @Override + public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context) { + return context.getJobConf(); + } + + @Override public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { boolean origDisableHDFSCache = conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false); diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 7629c2d..dac05e6 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -774,6 +774,11 @@ public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext con } @Override + public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context) { + return context.getJobConf(); + } + + @Override public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { return FileSystem.newInstance(uri, conf); } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index d72e0da..d1cbd2e 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -708,6 +708,13 @@ public void setFullFileStatus(Configuration conf, HdfsFileStatus sourceStatus, */ public Configuration getConfiguration(JobContext context); + /** + * Get job conf from the old style JobContext. + * @param context job context + * @return job conf + */ + public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context); + public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException; public void getMergedCredentials(JobConf jobConf) throws IOException;