Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (working copy) @@ -36,13 +36,13 @@ } public List run(String user) - throws NotAuthorizedException, BadParam, IOException { + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); TempletonJobTracker tracker = null; try { - tracker = new TempletonJobTracker(ugi, - JobTracker.getAddress(appConf), - appConf); + tracker = new TempletonJobTracker(JobTracker.getAddress(appConf), + appConf); ArrayList ids = new ArrayList(); Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (working copy) @@ -33,14 +33,13 @@ } public QueueStatusBean run(String user, String id) - throws NotAuthorizedException, BadParam, IOException + throws NotAuthorizedException, BadParam, IOException, InterruptedException { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); TempletonJobTracker tracker = null; JobState state = null; try { - tracker = new TempletonJobTracker(ugi, - JobTracker.getAddress(appConf), + tracker = new TempletonJobTracker(JobTracker.getAddress(appConf), appConf); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java (working copy) @@ -104,7 +104,12 @@ List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); if (tokenFile != null) { - jarArgsList.add(1, "-Dmapreduce.job.credentials.binary=" + tokenFile); + /* + * The magic number 3 comes from the fact that the -D option can + * be only after the jar command line option but before other + * options. + */ + jarArgsList.add(3, "-Dmapreduce.job.credentials.binary=" + tokenFile); } return execService.run(jarArgsList, removeEnv, env); } Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java (working copy) @@ -24,6 +24,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; +import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -34,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; /** @@ -213,12 +215,24 @@ if (fname == null || conf == null) { return null; } - FileSystem defaultFs = FileSystem.get(new URI(fname), conf, user); + + final Configuration fConf = new Configuration(conf); + final String finalFName = new String(fname); + + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + final FileSystem defaultFs = + ugi.doAs(new PrivilegedExceptionAction() { + public FileSystem run() + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException { + return FileSystem.get(new URI(finalFName), fConf); + } + }); + URI u = new URI(fname); Path p = new Path(u).makeQualified(defaultFs); - FileSystem fs = p.getFileSystem(conf); - if (hadoopFsIsMissing(fs, p)) + if (hadoopFsIsMissing(defaultFs, p)) throw new FileNotFoundException("File " + fname + " does not exist."); return p; Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (working copy) @@ -61,7 +61,7 @@ */ public class AppConfig extends Configuration { public static final String[] HADOOP_CONF_FILENAMES = { - "core-default.xml", "core-site.xml", "mapred-default.xml", "mapred-site.xml" + "core-default.xml", "core-site.xml", "mapred-default.xml", "mapred-site.xml", "hdfs-site.xml" }; public static final String[] HADOOP_PREFIX_VARS = { Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (working copy) @@ -678,7 +678,8 @@ @Path("queue/{jobid}") @Produces({MediaType.APPLICATION_JSON}) public QueueStatusBean showQueueId(@PathParam("jobid") String jobid) - throws NotAuthorizedException, BadParam, IOException { + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + verifyUser(); verifyParam(jobid, ":jobid"); @@ -693,7 +694,8 @@ @Path("queue/{jobid}") @Produces({MediaType.APPLICATION_JSON}) public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) - throws NotAuthorizedException, BadParam, IOException { + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + verifyUser(); verifyParam(jobid, ":jobid"); @@ -708,7 +710,8 @@ @Path("queue") @Produces({MediaType.APPLICATION_JSON}) public List showQueueList() - throws NotAuthorizedException, BadParam, IOException { + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + verifyUser(); ListDelegator d = new ListDelegator(appConf); Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (working copy) @@ -26,7 +26,6 @@ import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.JobTracker; import org.apache.hadoop.mapred.TempletonJobTracker; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hcatalog.templeton.tool.JobState; /** @@ -40,14 +39,13 @@ } public QueueStatusBean run(String user, String id) - throws NotAuthorizedException, BadParam, IOException { - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + throws NotAuthorizedException, BadParam, IOException, InterruptedException + { TempletonJobTracker tracker = null; JobState state = null; try { - tracker = new TempletonJobTracker(ugi, - JobTracker.getAddress(appConf), - appConf); + tracker = new TempletonJobTracker(JobTracker.getAddress(appConf), + appConf); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) throw new BadParam("Invalid jobid: " + id); Index: webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (working copy) @@ -146,12 +146,33 @@ ServletContextHandler root = new ServletContextHandler(server, "/"); // Add the Auth filter - root.addFilter(makeAuthFilter(), "/*", FilterMapping.REQUEST); + FilterHolder fHolder = makeAuthFilter(); + /* + * We add filters for each of the URIs supported by templeton. + * If we added the entire sub-structure using '/*', the mapreduce + * notification cannot give the callback to templeton in secure mode. + * This is because mapreduce does not use secure credentials for + * callbacks. So jetty would fail the request as unauthorized. + */ + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", + FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", + FilterMapping.REQUEST); + // Connect Jersey ServletHolder h = new ServletHolder(new ServletContainer(makeJerseyConfig())); root.addServlet(h, "/" + SERVLET_PATH + "/*"); - // Add any redirects addRedirects(server); Index: webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java =================================================================== --- webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (revision 1387376) +++ webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; @@ -29,23 +30,29 @@ * Communicate with the JobTracker as a specific user. */ public class TempletonJobTracker { - private JobSubmissionProtocol cnx; + private final JobSubmissionProtocol cnx; /** * Create a connection to the Job Tracker. */ - public TempletonJobTracker(UserGroupInformation ugi, - InetSocketAddress addr, - Configuration conf) - throws IOException { - cnx = (JobSubmissionProtocol) - RPC.getProxy(JobSubmissionProtocol.class, - JobSubmissionProtocol.versionID, - addr, - ugi, - conf, - NetUtils.getSocketFactory(conf, - JobSubmissionProtocol.class)); + public TempletonJobTracker(final InetSocketAddress addr, + final Configuration conf) + throws IOException, InterruptedException { + + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + cnx = + ugi.doAs(new PrivilegedExceptionAction() { + public JobSubmissionProtocol run () + throws IOException, InterruptedException { + return (JobSubmissionProtocol) + RPC.getProxy(JobSubmissionProtocol.class, + JobSubmissionProtocol.versionID, + addr, + conf, + NetUtils.getSocketFactory(conf, + JobSubmissionProtocol.class)); + } + }); } /**