diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java index 4b2dfec..622f92d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.security.UserGroupInformation; @@ -41,10 +42,11 @@ public DeleteDelegator(AppConfig appConf) { public QueueStatusBean run(String user, String id) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - UserGroupInformation ugi = UgiFactory.getUgi(user); + UserGroupInformation ugi = null; WebHCatJTShim tracker = null; JobState state = null; try { + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) @@ -69,6 +71,8 @@ public QueueStatusBean run(String user, String id) tracker.close(); if (state != null) state.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } } } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java index 1455316..9bea897 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.shims.HadoopShimsSecure; @@ -178,8 +179,9 @@ public EnqueueBean enqueueJob(String user, Map userArgs, String List args, TempletonControllerJob controllerJob) throws NotAuthorizedException, BusyException, IOException, QueueException { + UserGroupInformation ugi = null; try { - UserGroupInformation ugi = UgiFactory.getUgi(user); + ugi = UgiFactory.getUgi(user); final long startTime = System.nanoTime(); @@ -197,6 +199,10 @@ public EnqueueBean enqueueJob(String user, Map userArgs, String return new EnqueueBean(id); } catch (InterruptedException e) { throw new QueueException("Unable to launch job " + e); + } finally { + if (ugi != null) { + FileSystem.closeAllForUGI(ugi); + } } } @@ -344,24 +350,35 @@ protected void killJob(String user, String jobId) */ private String getShimLibjars() { WebHCatJTShim shim = null; + UserGroupInformation ugi = null; try { - shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, UserGroupInformation.getCurrentUser()); + ugi = UserGroupInformation.getCurrentUser(); + shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); + + // Besides the HiveShims jar which is Hadoop version dependent we also + // always need to include hive shims common jars. + Path shimCommonJar = new Path( + TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN)); + Path shimCommonSecureJar = new Path( + TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN)); + Path shimJar = new Path( + TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN)); + + return String.format( + "%s,%s,%s", + shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString()); } catch (IOException e) { - throw new RuntimeException("Failed to get WebHCatShim", e); + throw new RuntimeException("Failed to get shimLibJars", e); + } finally { + try { + if (ugi != null) { + FileSystem.closeAllForUGI(ugi); + } + } catch (IOException e) { + throw new RuntimeException("Failed to closeAllForUGI", e); + } } - // Besides the HiveShims jar which is Hadoop version dependent we also - // always need to include hive shims common jars. - Path shimCommonJar = new Path( - TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN)); - Path shimCommonSecureJar = new Path( - TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN)); - Path shimJar = new Path( - TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN)); - - return String.format( - "%s,%s,%s", - shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString()); } // Storage vars diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java index ed4cea9..dfa59f8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobStatus; @@ -104,11 +105,12 @@ public ListDelegator(AppConfig appConf) { int numRecords, boolean showDetails) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - UserGroupInformation ugi = UgiFactory.getUgi(user); + UserGroupInformation ugi = null; WebHCatJTShim tracker = null; ArrayList ids = new ArrayList(); try { + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobStatus[] jobs = tracker.getAllJobs(); @@ -125,6 +127,8 @@ public ListDelegator(AppConfig appConf) { } finally { if (tracker != null) tracker.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } return getJobStatus(ids, user, showall, jobId, numRecords, showDetails); diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java index 5aed3b3..793881b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java @@ -170,6 +170,7 @@ public Object run() throws IOException, URISyntaxException { return null; } }); + FileSystem.closeAllForUGI(ugi); return twrapper.tokens; } private static void collectTokens(FileSystem fs, TokenWrapper twrapper, Credentials creds, String userName) throws IOException { @@ -204,6 +205,7 @@ public Object run() throws IOException { return null; } }); + FileSystem.closeAllForUGI(ugi); } @@ -220,6 +222,7 @@ public String run() return client.getDelegationToken(c.getUser(), u); } }); + FileSystem.closeAllForUGI(ugi); return s; } } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java index 4112eef..c042ae8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.mapred.JobID; @@ -112,8 +113,9 @@ public QueueStatusBean getJobStatus(String user, String id) { WebHCatJTShim tracker = null; JobState state = null; + UserGroupInformation ugi = null; try { - UserGroupInformation ugi = UgiFactory.getUgi(user); + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) @@ -127,6 +129,8 @@ public QueueStatusBean getJobStatus(String user, String id) tracker.close(); if (state != null) state.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index 07b005b..e0ccc70 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -362,6 +362,7 @@ public FileSystem run() if (hadoopFsIsMissing(defaultFs, p)) throw new FileNotFoundException("File " + fname + " does not exist."); + FileSystem.closeAllForUGI(ugi); return p; }