diff --git conf/templeton-default.xml conf/templeton-default.xml
index bdd86d0..2db7605 100644
--- conf/templeton-default.xml
+++ conf/templeton-default.xml
@@ -78,6 +78,12 @@ limitations under the License.
+ templeton.python
+ ${env.PYTHON_CMD}
+ The path to the python executable.
+
+
+
templeton.pig.archive
hdfs:///user/templeton/pig-0.9.2.tar.gz
The path to the Pig archive.
diff --git conf/templeton-env.sh conf/templeton-env.sh
new file mode 100644
index 0000000..2489eec
--- /dev/null
+++ conf/templeton-env.sh
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export JAVA_HOME=${JAVA_HOME}
+
+export PYTHON_CMD=`which python`
diff --git src/test/e2e/templeton/build.xml src/test/e2e/templeton/build.xml
index ed85eba..64b5180 100644
--- src/test/e2e/templeton/build.xml
+++ src/test/e2e/templeton/build.xml
@@ -17,76 +17,89 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git src/test/e2e/templeton/drivers/TestDriverCurl.pm src/test/e2e/templeton/drivers/TestDriverCurl.pm
index 79373bf..3591397 100644
--- src/test/e2e/templeton/drivers/TestDriverCurl.pm
+++ src/test/e2e/templeton/drivers/TestDriverCurl.pm
@@ -155,6 +155,9 @@ sub globalSetup
# Setup the output path
my $me = `whoami`;
chomp $me;
+ #usernames on windows can be "domain\username" change the "\"
+ # as runid is used in file names
+ $me =~ s/\\/_/;
$globalHash->{'runid'} = $me . "." . time;
# if "-ignore false" was provided on the command line,
@@ -495,7 +498,7 @@ sub execCurlCmd(){
push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header);
push @curl_cmd, ($url);
- print $log "$0:$subName Going to run command : " . join (' ', @curl_cmd);
+ print $log "$0:$subName Going to run command : " . join (' , ', @curl_cmd);
print $log "\n";
diff --git src/test/e2e/templeton/tests/ddl.conf src/test/e2e/templeton/tests/ddl.conf
index 673b9f2..36c15f9 100644
--- src/test/e2e/templeton/tests/ddl.conf
+++ src/test/e2e/templeton/tests/ddl.conf
@@ -337,7 +337,7 @@ $cfg =
"fieldsTerminatedBy" : "\u0001",
"collectionItemsTerminatedBy" : "\u0002",
"mapKeysTerminatedBy" : "\u0003",
- "linesTerminatedBy" : "\n",
+ "linesTerminatedBy" : "\\\n",
"serde" : {
"name" : "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe",
@@ -500,15 +500,7 @@ $cfg =
'url' => ':TEMPLETON_URL:/templeton/v1/ddl',
'status_code' => 200,
'post_options' => ['user.name=:UNAME:',
- 'exec=create table if not exists templetontest_parts (i int, j bigint, ip STRING COMMENT "IP Address of the User")
-COMMENT "This is the page view table"
- PARTITIONED BY(dt STRING, country STRING)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY "\001"
- COLLECTION ITEMS TERMINATED BY "\002"
- MAP KEYS TERMINATED BY "\003"
-STORED AS rcfile
---LOCATION "table1_location" '],
+ 'exec=create table if not exists templetontest_parts (i int, j bigint, ip STRING COMMENT \'IP Address of the User\') COMMENT \'This is the page view table\' PARTITIONED BY(dt STRING, country STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY \'\001\' COLLECTION ITEMS TERMINATED BY \'\002\' MAP KEYS TERMINATED BY \'\003\' STORED AS rcfile --LOCATION \'table1_location\' '],
'json_field_substr_match' => {'stderr' => 'OK'},
},
{
@@ -747,7 +739,7 @@ STORED AS rcfile
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl?user.name=:UNAME:',
'status_code' => 200,
- 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testcol_tab (i int comment "column with comment", j bigint) STORED AS rcfile;'],
+ 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testcol_tab (i int comment \'column with comment\', j bigint) STORED AS rcfile;'],
'json_field_substr_match' => {'stderr' => 'OK'},
},
{
@@ -1074,9 +1066,7 @@ STORED AS rcfile
'status_code' => 200,
'post_options' => ['user.name=:UNAME:',
'permissions=---------',
- 'exec=create table templetontest_hcatgp(i int, j bigint)
- PARTITIONED BY(dt STRING, country STRING)
- STORED AS rcfile;'
+ 'exec=create table templetontest_hcatgp(i int, j bigint) PARTITIONED BY(dt STRING, 3Bcountry STRING) STORED AS rcfile;'
],
'json_field_substr_match' => {'stderr' => 'OK', 'exitcode' => '^0$'}
},
diff --git src/test/e2e/templeton/tests/jobsubmission.conf src/test/e2e/templeton/tests/jobsubmission.conf
index 6e4ebed..69a8dea 100644
--- src/test/e2e/templeton/tests/jobsubmission.conf
+++ src/test/e2e/templeton/tests/jobsubmission.conf
@@ -18,42 +18,6 @@ $cfg =
[
##=============================================================================================================
{
- 'name' => 'TestStreaming',
- 'tests' =>
- [
- {
- 'num' => 1,
- 'method' => 'POST',
- 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
- 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/cat', 'reducer=/usr/bin/wc'],
- 'json_field_substr_match' => { 'id' => '\d+'},
- #results
- 'status_code' => 200,
- 'check_job_created' => 1,
- 'check_job_complete' => 'SUCCESS',
- 'check_call_back' => 1,
- },
- {
- #-ve test - no input file
- 'num' => 2,
- 'ignore' => 'wait for fix in hadoop 1.0.3',
- 'method' => 'POST',
- 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
- 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/ls no_such-file-12e3', 'reducer=/usr/bin/wc'],
- 'json_field_substr_match' => { 'id' => '\d+'},
- #results
- 'status_code' => 200,
- 'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
- 'check_call_back' => 1,
- },
-
- ]
- },
-##=============================================================================================================
- {
'name' => 'TestKillJob',
'tests' =>
[
@@ -62,7 +26,7 @@ $cfg =
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/sleep 100', 'reducer=/usr/bin/wc'],
+ 'mapper=\cygwin\bin\sleep 100', 'reducer=\cygwin\bin\wc'],
'json_field_substr_match' => { 'id' => '\d+'},
#results
'status_code' => 200,
diff --git src/test/e2e/templeton/tests/jobsubmission_streaming_lin.conf src/test/e2e/templeton/tests/jobsubmission_streaming_lin.conf
new file mode 100644
index 0000000..ec0107d
--- /dev/null
+++ src/test/e2e/templeton/tests/jobsubmission_streaming_lin.conf
@@ -0,0 +1,60 @@
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'TestStreaming',
+ 'tests' =>
+ [
+ {
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=/bin/cat', 'reducer=/usr/bin/wc'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #-ve test - no input file
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=/bin/ls no_such-file-12e3', 'reducer=/usr/bin/wc'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'FAILURE',
+ 'check_call_back' => 1,
+ },
+
+ ]
+ },
+##=============================================================================================================
+
+
+ ]
+},
+ ;
+
diff --git src/test/e2e/templeton/tests/jobsubmission_streaming_win.conf src/test/e2e/templeton/tests/jobsubmission_streaming_win.conf
new file mode 100644
index 0000000..6328d78
--- /dev/null
+++ src/test/e2e/templeton/tests/jobsubmission_streaming_win.conf
@@ -0,0 +1,60 @@
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'TestStreaming',
+ 'tests' =>
+ [
+ {
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=\cygwin\bin\cat.exe', 'reducer=\cygwin\bin\wc.exe'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #-ve test - no input file
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=/bin/ls no_such-file-12e3', 'reducer=/usr/bin/wc'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'FAILURE',
+ 'check_call_back' => 1,
+ },
+
+ ]
+ },
+##=============================================================================================================
+
+
+ ]
+},
+ ;
+
diff --git webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
index 80ddb1e..f39a1a8 100644
--- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
+++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -84,6 +84,7 @@ public class AppConfig extends Configuration {
public static final String HADOOP_NAME = "templeton.hadoop";
public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir";
public static final String HCAT_NAME = "templeton.hcat";
+ public static final String PYTHON_NAME = "templeton.python";
public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
public static final String HIVE_PATH_NAME = "templeton.hive.path";
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
@@ -168,6 +169,7 @@ public class AppConfig extends Configuration {
public String libJars() { return get(LIB_JARS_NAME); }
public String clusterHadoop() { return get(HADOOP_NAME); }
public String clusterHcat() { return get(HCAT_NAME); }
+ public String clusterPython() { return get(PYTHON_NAME); }
public String pigPath() { return get(PIG_PATH_NAME); }
public String pigArchive() { return get(PIG_ARCHIVE_NAME); }
public String hivePath() { return get(HIVE_PATH_NAME); }
@@ -192,14 +194,11 @@ public class AppConfig extends Configuration {
public long zkCleanupInterval() {
return getLong(ZooKeeperCleanup.ZK_CLEANUP_INTERVAL,
- (1000L * 60L * 60L * 12L));
- }
-
- public long zkMaxAge() {
+ (1000L * 60L * 60L * 12L)); }
+ public long zkMaxAge() {
return getLong(ZooKeeperCleanup.ZK_CLEANUP_MAX_AGE,
- (1000L * 60L * 60L * 24L * 7L));
- }
-
+ (1000L * 60L * 60L * 24L * 7L)); }
public String zkHosts() { return get(ZooKeeperStorage.ZK_HOSTS); }
- public int zkSessionTimeout() { return getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT, 30000); }
+ public int zkSessionTimeout() { return getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT,
+ 30000); }
}
diff --git webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
index 41920b5..edbd36a 100644
--- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
+++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,12 +17,18 @@
*/
package org.apache.hcatalog.templeton;
+import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.Semaphore;
import org.apache.commons.exec.CommandLine;
@@ -32,6 +38,39 @@ import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+
+
+class StreamOutputWriter extends Thread
+{
+ InputStream is;
+ String type;
+ PrintWriter out;
+
+ StreamOutputWriter(InputStream is, String type, OutputStream outStream)
+ {
+ this.is = is;
+ this.type = type;
+ this.out = new PrintWriter(outStream, true);
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ BufferedReader br =
+ new BufferedReader(new InputStreamReader(is));
+ String line = null;
+ while ( (line = br.readLine()) != null){
+ out.println(line);
+ }
+ } catch (IOException ioe)
+ {
+ ioe.printStackTrace();
+ }
+ }
+}
/**
* Execute a local program. This is a singleton service that will
@@ -65,14 +104,15 @@ public class ExecServiceImpl implements ExecService {
* the number of processes that can simultaneously created for
* this instance.
*
+ * @param user A valid user
* @param program The program to run
- * @param args Arguments to pass to the program
* @param env Any extra environment variables to set
- * @return The result of the run.
+ * @returns The result of the run.
*/
public ExecBean run(String program, List args,
Map env)
- throws NotAuthorizedException, BusyException, ExecuteException, IOException {
+ throws NotAuthorizedException, BusyException, ExecuteException, IOException
+ {
boolean aquired = false;
try {
aquired = avail.tryAcquire();
@@ -92,14 +132,14 @@ public class ExecServiceImpl implements ExecService {
* Run the program synchronously as the given user. Warning:
* CommandLine will trim the argument strings.
*
+ * @param user A valid user
* @param program The program to run.
- * @param args Arguments to pass to the program
- * @param env Any extra environment variables to set
- * @return The result of the run.
+ * @returns The result of the run.
*/
public ExecBean runUnlimited(String program, List args,
Map env)
- throws NotAuthorizedException, ExecuteException, IOException {
+ throws NotAuthorizedException, ExecuteException, IOException
+ {
try {
return auxRun(program, args, env);
} catch (IOException e) {
@@ -108,12 +148,13 @@ public class ExecServiceImpl implements ExecService {
throw e;
else
throw new IOException("Invalid permissions on Templeton directory: "
- + cwd.getCanonicalPath());
+ + cwd.getCanonicalPath());
}
}
private ExecBean auxRun(String program, List args, Map env)
- throws NotAuthorizedException, ExecuteException, IOException {
+ throws NotAuthorizedException, ExecuteException, IOException
+ {
DefaultExecutor executor = new DefaultExecutor();
executor.setExitValues(null);
@@ -132,17 +173,54 @@ public class ExecServiceImpl implements ExecService {
LOG.info("Running: " + cmd);
ExecBean res = new ExecBean();
- res.exitcode = executor.execute(cmd, execEnv(env));
+
+ if(Shell.WINDOWS){
+ //The default executor is sometimes causing failure on windows. hcat
+ // command sometimes returns non zero exit status with it. It seems
+ // to hit some race conditions on windows.
+ env = execEnv(env);
+ String[] envVals = new String[env.size()];
+ int i=0;
+ for( Entry kv : env.entrySet()){
+ envVals[i++] = kv.getKey() + "=" + kv.getValue();
+ System.out.println("Setting " + kv.getKey() + "=" + kv.getValue());
+ }
+ Process proc = Runtime.getRuntime().exec(cmd.toStrings(), envVals);
+ //consume stderr
+ StreamOutputWriter errorGobbler = new
+ StreamOutputWriter(proc.getErrorStream(), "ERROR", errStream);
+
+ //consume stdout
+ StreamOutputWriter outputGobbler = new
+ StreamOutputWriter(proc.getInputStream(), "OUTPUT", outStream);
+
+ //start collecting input streams
+ errorGobbler.start();
+ outputGobbler.start();
+ //execute
+ try{
+ res.exitcode = proc.waitFor();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ //flush
+ errorGobbler.out.flush();
+ outputGobbler.out.flush();
+ }
+ else {
+ res.exitcode = executor.execute(cmd, execEnv(env));
+ }
String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
res.stdout = outStream.toString(enc);
res.stderr = errStream.toString(enc);
-
return res;
+
}
private CommandLine makeCommandLine(String program,
List args)
- throws NotAuthorizedException, IOException {
+ throws NotAuthorizedException, IOException
+ {
String path = validateProgram(program);
CommandLine cmd = new CommandLine(path);
if (args != null)
@@ -166,9 +244,10 @@ public class ExecServiceImpl implements ExecService {
res.put(key, val);
}
}
+
if (env != null)
res.putAll(env);
- for (Map.Entry envs : res.entrySet()) {
+ for(Map.Entry envs : res.entrySet()){
LOG.info("Env " + envs.getKey() + "=" + envs.getValue());
}
return res;
@@ -179,10 +258,11 @@ public class ExecServiceImpl implements ExecService {
* an exception if the program is missing or not authorized.
*
* @param path The path of the program.
- * @return The path of the validated program.
+ * @return The path of the validated program.
*/
public String validateProgram(String path)
- throws NotAuthorizedException, IOException {
+ throws NotAuthorizedException, IOException
+ {
File f = new File(path);
if (f.canExecute()) {
return f.getCanonicalPath();
diff --git webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java
index 94855dd..cca677e 100644
--- webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java
+++ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+
import javax.ws.rs.core.Response;
import org.apache.commons.exec.ExecuteException;
@@ -34,7 +35,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hcatalog.templeton.tool.TempletonUtils;
-
/**
* Run hcat on the local server using the ExecService. This is
* the backend of the ddl web service.
@@ -53,7 +53,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public ExecBean run(String user, String exec, boolean format,
String group, String permissions)
- throws NotAuthorizedException, BusyException, ExecuteException, IOException {
+ throws NotAuthorizedException, BusyException, ExecuteException, IOException
+ {
SecureProxySupport proxy = new SecureProxySupport();
try {
List args = makeArgs(exec, format, group, permissions);
@@ -64,7 +65,7 @@ public class HcatDelegator extends LauncherDelegator {
Map env = TempletonUtils.hadoopUserEnv(user, cp);
proxy.addEnv(env);
proxy.addArgs(args);
- return execService.run(appConf.clusterHcat(), args, env);
+ return execService.run(appConf.clusterPython(), args, env);
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
@@ -76,8 +77,9 @@ public class HcatDelegator extends LauncherDelegator {
private List makeArgs(String exec, boolean format,
String group, String permissions) {
ArrayList args = new ArrayList();
+ args.add(appConf.clusterHcat());
args.add("-e");
- args.add(exec);
+ args.add('"' + exec + '"');
if (TempletonUtils.isset(group)) {
args.add("-g");
args.add(group);
@@ -103,7 +105,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response descDatabase(String user, String db, boolean extended)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "desc database " + db + "; ";
if (extended)
exec = "desc database extended " + db + "; ";
@@ -114,12 +117,12 @@ public class HcatDelegator extends LauncherDelegator {
} catch (HcatException e) {
if (e.execBean.stderr.indexOf("Error in semantic analysis") > -1) {
return JsonBuilder.create().
- put("error", "Database " + db + " does not exist")
- .put("errorCode", "404")
- .put("database", db).build();
+ put("error", "Database " + db + " does not exist")
+ .put("errorCode", "404")
+ .put("database", db).build();
}
throw new HcatException("unable to describe database: " + db,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -129,7 +132,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response listDatabases(String user, String dbPattern)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("show databases like '%s';", dbPattern);
try {
String res = jsonRun(user, exec);
@@ -137,7 +141,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to show databases for: " + dbPattern,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -146,7 +150,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response createDatabase(String user, DatabaseDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "create database";
if (desc.ifNotExists)
exec += " if not exists";
@@ -157,7 +162,7 @@ public class HcatDelegator extends LauncherDelegator {
exec += String.format(" location '%s'", desc.location);
if (TempletonUtils.isset(desc.properties))
exec += String.format(" with dbproperties (%s)",
- makePropertiesStatement(desc.properties));
+ makePropertiesStatement(desc.properties));
exec += ";";
String res = jsonRun(user, exec, desc.group, desc.permissions);
@@ -173,7 +178,8 @@ public class HcatDelegator extends LauncherDelegator {
boolean ifExists, String option,
String group, String permissions)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "drop database";
if (ifExists)
exec += " if exists";
@@ -193,7 +199,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response createTable(String user, String db, TableDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = makeCreateTable(db, desc);
try {
@@ -205,7 +212,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (final HcatException e) {
throw new HcatException("unable to create table: " + desc.table,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -214,7 +221,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response createTableLike(String user, String db, TableLikeDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; create", db);
if (desc.external)
@@ -233,7 +241,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (final HcatException e) {
throw new HcatException("unable to create table: " + desc.newTable,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -242,7 +250,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response descTable(String user, String db, String table, boolean extended)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "use " + db + "; ";
if (extended)
exec += "desc extended " + table + "; ";
@@ -256,7 +265,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to describe table: " + table,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -266,9 +275,10 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response listTables(String user, String db, String tablePattern)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; show tables like '%s';",
- db, tablePattern);
+ db, tablePattern);
try {
String res = jsonRun(user, exec);
return JsonBuilder.create(res)
@@ -276,7 +286,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to show tables for: " + tablePattern,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -286,9 +296,10 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response descExtendedTable(String user, String db, String table)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; show table extended like %s;",
- db, table);
+ db, table);
try {
String res = jsonRun(user, exec);
JsonBuilder jb = JsonBuilder.create(singleTable(res, table))
@@ -355,7 +366,7 @@ public class HcatDelegator extends LauncherDelegator {
exec += String.format(" location '%s'", desc.location);
if (TempletonUtils.isset(desc.tableProperties))
exec += String.format(" tblproperties (%s)",
- makePropertiesStatement(desc.tableProperties));
+ makePropertiesStatement(desc.tableProperties));
exec += ";";
return exec;
@@ -402,9 +413,9 @@ public class HcatDelegator extends LauncherDelegator {
private String makeRowFormat(TableDesc.RowFormatDesc desc) {
String res =
makeTermBy(desc.fieldsTerminatedBy, "fields")
- + makeTermBy(desc.collectionItemsTerminatedBy, "collection items")
- + makeTermBy(desc.mapKeysTerminatedBy, "map keys")
- + makeTermBy(desc.linesTerminatedBy, "lines");
+ + makeTermBy(desc.collectionItemsTerminatedBy, "collection items")
+ + makeTermBy(desc.mapKeysTerminatedBy, "map keys")
+ + makeTermBy(desc.linesTerminatedBy, "lines");
if (TempletonUtils.isset(res))
return "row format delimited" + res;
@@ -418,7 +429,7 @@ public class HcatDelegator extends LauncherDelegator {
private String makeTermBy(String sep, String fieldName) {
if (TempletonUtils.isset(sep))
- return String.format(" %s terminated by '%s'", fieldName, sep);
+ return String.format(" %s terminated by '%s'", fieldName, sep);
else
return "";
}
@@ -428,7 +439,7 @@ public class HcatDelegator extends LauncherDelegator {
String res = "row format serde " + desc.name;
if (TempletonUtils.isset(desc.properties))
res += String.format(" with serdeproperties (%s)",
- makePropertiesStatement(desc.properties));
+ makePropertiesStatement(desc.properties));
return res;
}
@@ -445,13 +456,14 @@ public class HcatDelegator extends LauncherDelegator {
String res = String.format("stored by '%s'", desc.className);
if (TempletonUtils.isset(desc.properties))
res += String.format(" with serdeproperties (%s)",
- makePropertiesStatement(desc.properties));
+ makePropertiesStatement(desc.properties));
return res;
}
// Pull out the first table from the "show extended" json.
private String singleTable(String json, String table)
- throws IOException {
+ throws IOException
+ {
Map obj = JsonBuilder.jsonToMap(json);
if (JsonBuilder.isError(obj))
return json;
@@ -462,8 +474,8 @@ public class HcatDelegator extends LauncherDelegator {
else {
return JsonBuilder
.createError(String.format("Table %s does not exist", table),
- JsonBuilder.MISSING).
- buildJson();
+ JsonBuilder.MISSING).
+ buildJson();
}
}
@@ -474,7 +486,8 @@ public class HcatDelegator extends LauncherDelegator {
String table, boolean ifExists,
String group, String permissions)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; drop table", db);
if (ifExists)
exec += " if exists";
@@ -498,9 +511,10 @@ public class HcatDelegator extends LauncherDelegator {
String oldTable, String newTable,
String group, String permissions)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; alter table %s rename to %s;",
- db, oldTable, newTable);
+ db, oldTable, newTable);
try {
String res = jsonRun(user, exec, group, permissions, true);
return JsonBuilder.create(res)
@@ -509,7 +523,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to rename table: " + oldTable,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -519,7 +533,8 @@ public class HcatDelegator extends LauncherDelegator {
public Response descTableProperty(String user, String db,
String table, String property)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
Response res = descTable(user, db, table, true);
if (res.getStatus() != JsonBuilder.OK)
return res;
@@ -545,7 +560,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response listTableProperties(String user, String db, String table)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
Response res = descTable(user, db, table, true);
if (res.getStatus() != JsonBuilder.OK)
return res;
@@ -563,10 +579,11 @@ public class HcatDelegator extends LauncherDelegator {
public Response addOneTableProperty(String user, String db, String table,
TablePropertyDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec
= String.format("use %s; alter table %s set tblproperties ('%s'='%s');",
- db, table, desc.name, desc.value);
+ db, table, desc.name, desc.value);
try {
String res = jsonRun(user, exec, desc.group, desc.permissions, true);
return JsonBuilder.create(res)
@@ -576,12 +593,12 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to add table property: " + table,
- e.execBean, exec);
+ e.execBean, exec);
}
}
private Map tableProperties(Object extendedTable) {
- if (!(extendedTable instanceof Map))
+ if (! (extendedTable instanceof Map))
return null;
Map m = (Map) extendedTable;
Map tableInfo = (Map) m.get("tableInfo");
@@ -596,7 +613,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response listPartitions(String user, String db, String table)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "use " + db + "; ";
exec += "show partitions " + table + "; ";
try {
@@ -607,7 +625,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to show partitions for table: " + table,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -617,7 +635,8 @@ public class HcatDelegator extends LauncherDelegator {
public Response descOnePartition(String user, String db, String table,
String partition)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = "use " + db + "; ";
exec += "show table extended like " + table
+ " partition (" + partition + "); ";
@@ -631,9 +650,9 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to show partition: "
- + table + " " + partition,
- e.execBean,
- exec);
+ + table + " " + partition,
+ e.execBean,
+ exec);
}
}
@@ -643,7 +662,8 @@ public class HcatDelegator extends LauncherDelegator {
public Response addOnePartition(String user, String db, String table,
PartitionDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; alter table %s add", db, table);
if (desc.ifNotExists)
exec += " if not exists";
@@ -655,11 +675,11 @@ public class HcatDelegator extends LauncherDelegator {
String res = jsonRun(user, exec, desc.group, desc.permissions, true);
if (res.indexOf("AlreadyExistsException") > -1) {
return JsonBuilder.create().
- put("error", "Partition already exists")
- .put("errorCode", "409")
- .put("database", db)
- .put("table", table)
- .put("partition", desc.partition).build();
+ put("error", "Partition already exists")
+ .put("errorCode", "409")
+ .put("database", db)
+ .put("table", table)
+ .put("partition", desc.partition).build();
}
return JsonBuilder.create(res)
.put("database", db)
@@ -668,7 +688,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to add partition: " + desc,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -679,7 +699,8 @@ public class HcatDelegator extends LauncherDelegator {
String table, String partition, boolean ifExists,
String group, String permissions)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
String exec = String.format("use %s; alter table %s drop", db, table);
if (ifExists)
exec += " if exists";
@@ -694,7 +715,7 @@ public class HcatDelegator extends LauncherDelegator {
.build();
} catch (HcatException e) {
throw new HcatException("unable to drop partition: " + partition,
- e.execBean, exec);
+ e.execBean, exec);
}
}
@@ -704,12 +725,13 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response listColumns(String user, String db, String table)
throws HcatException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
try {
return descTable(user, db, table, false);
} catch (HcatException e) {
throw new HcatException("unable to show columns for table: " + table,
- e.execBean, e.statement);
+ e.execBean, e.statement);
}
}
@@ -718,7 +740,8 @@ public class HcatDelegator extends LauncherDelegator {
*/
public Response descOneColumn(String user, String db, String table, String column)
throws SimpleWebException, NotAuthorizedException, BusyException,
- ExecuteException, IOException {
+ ExecuteException, IOException
+ {
Response res = listColumns(user, db, table);
if (res.getStatus() != JsonBuilder.OK)
return res;
@@ -727,7 +750,7 @@ public class HcatDelegator extends LauncherDelegator {
final Map fields = (o != null && (o instanceof Map)) ? (Map) o : null;
if (fields == null)
throw new SimpleWebException(500, "Internal error, unable to find column "
- + column);
+ + column);
List