diff --git hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hcatalog/webhcat/svr/src/main/config/webhcat-default.xml index d00f728..3f3430e 100644 --- hcatalog/webhcat/svr/src/main/config/webhcat-default.xml +++ hcatalog/webhcat/svr/src/main/config/webhcat-default.xml @@ -116,6 +116,12 @@ + templeton.sqoop.path + ${env.SQOOP_HOME}/bin/sqoop.cmd + The path to the Sqoop executable. + + + templeton.exec.encoding UTF-8 The encoding of the stdout and stderr data. diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java index 8c143a8..29e3a6a 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java @@ -90,6 +90,8 @@ public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive"; public static final String HIVE_PATH_NAME = "templeton.hive.path"; public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; + public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive"; + public static final String SQOOP_PATH_NAME = "templeton.sqoop.path"; public static final String LIB_JARS_NAME = "templeton.libjars"; public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; public static final String PIG_PATH_NAME = "templeton.pig.path"; @@ -184,6 +186,8 @@ private boolean loadOneClasspathConfig(String fname) { public String pigArchive() { return get(PIG_ARCHIVE_NAME); } public String hivePath() { return get(HIVE_PATH_NAME); } public String hiveArchive() { return get(HIVE_ARCHIVE_NAME); } + public String sqoopPath() { return get(SQOOP_PATH_NAME); } + public String sqoopArchive() { return get(SQOOP_ARCHIVE_NAME); } public String streamingJar() { return get(STREAMING_JAR_NAME); } public String kerberosSecret() { return get(KERBEROS_SECRET); } public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java index 3761228..c4d8dcb 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java @@ -174,6 +174,8 @@ public Server runServer(int port) FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", + FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java index 29ac4b3..457e5e2 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java @@ -673,6 +673,28 @@ public EnqueueBean hive(@FormParam("execute") String execute, } /** + * Run a Sqoop job. + */ + @POST + @Path("sqoop") + @Produces({MediaType.APPLICATION_JSON}) + public EnqueueBean sqoop(@FormParam("command") String command, + @FormParam("file") String optionsFile, + @FormParam("files") String otherFiles, + @FormParam("statusdir") String statusdir, + @FormParam("callback") String callback) + throws NotAuthorizedException, BusyException, BadParam, QueueException, + ExecuteException, IOException, InterruptedException { + verifyUser(); + if (command == null && optionsFile == null) + throw new BadParam("Must define Sqoop command or a file contains Sqoop command to run Sqoop job"); + + SqoopDelegator d = new SqoopDelegator(appConf); + return d.run(getUser(), command, optionsFile, otherFiles, + statusdir, callback, getCompletedUrl()); + } + + /** * Return the status of the jobid. */ @GET diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SqoopDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SqoopDelegator.java new file mode 100644 index 0000000..e30c905 --- /dev/null +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SqoopDelegator.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.tool.TempletonControllerJob; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Submit a Sqoop job. + * + * This is the backend of the Sqoop web service. + */ +public class SqoopDelegator extends LauncherDelegator { + + public SqoopDelegator(AppConfig appConf) { + super(appConf); + } + + public EnqueueBean run(String user, String command, + String optionsFile, String otherFiles, String statusdir, + String callback, String completedUrl) + throws NotAuthorizedException, BadParam, BusyException, QueueException, + ExecuteException, IOException, InterruptedException + { + runAs = user; + List args = makeArgs(command, optionsFile, otherFiles, statusdir, + completedUrl); + + return enqueueController(user, callback, args); + } + + List makeArgs(String command, String optionsFile, String otherFiles, + String statusdir, String completedUrl) + throws BadParam, IOException, InterruptedException + { + ArrayList args = new ArrayList(); + try { + args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl)); + args.add("--"); + args.add(appConf.sqoopPath()); + if (TempletonUtils.isset(command)) { + String[] temArgs = command.split(" "); + for (int i = 0; i < temArgs.length; i++) { + args.add(temArgs[i]); + //the token file location should be right after the tool argument + if (i == 0 && !temArgs[i].startsWith("--")) { + args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); + } + } + } else if (TempletonUtils.isset(optionsFile)) { + args.add("--options-file"); + args.add(TempletonUtils.hadoopFsPath(optionsFile, appConf, runAs) + .getName()); + } + } catch (FileNotFoundException e) { + throw new BadParam(e.getMessage()); + } catch (URISyntaxException e) { + throw new BadParam(e.getMessage()); + } + return args; + } + + private List makeBasicArgs(String optionsFile, String otherFiles, + String statusdir, String completedUrl) + throws URISyntaxException, FileNotFoundException, IOException, + InterruptedException + { + ArrayList args = new ArrayList(); + + ArrayList allFiles = new ArrayList(); + if (TempletonUtils.isset(optionsFile)) + allFiles.add(TempletonUtils.hadoopFsFilename(optionsFile, appConf, + runAs)); + if (TempletonUtils.isset(otherFiles)) { + String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs); + allFiles.addAll(Arrays.asList(ofs)); + } + args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles)); + + if (appConf.sqoopArchive() != null && !appConf.sqoopArchive().equals("")) + { + args.add("-archives"); + args.add(appConf.sqoopArchive()); + } + + return args; + } +}