diff --git hcatalog/src/test/e2e/templeton/README.txt hcatalog/src/test/e2e/templeton/README.txt
index dac6ffc..647cfc1 100644
--- hcatalog/src/test/e2e/templeton/README.txt
+++ hcatalog/src/test/e2e/templeton/README.txt
@@ -150,3 +150,14 @@ Also useful to add to conf/hadoop-env.sh
export HADOOP_OPTS="-Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk"
to prevent warning about SCDynamicStore which may throw some tests off
(http://stackoverflow.com/questions/7134723/hadoop-on-osx-unable-to-load-realm-info-from-scdynamicstore)
+
+
+Performance
+-----------
+It's a good idea to set fork.factor.conf.file={number of .conf files} and fork.factor.group to something > 1
+(see build.xml) to make these tests run faster. If doing this, make sure the Hadoop Cluster has
+enough map slots (10?) (mapred.tasktracker.map.tasks.maximum), otherwise test parallelism won't help.
+
+Adding Tests
+------------
+ToDo: add some guidelines
diff --git hcatalog/src/test/e2e/templeton/build.xml hcatalog/src/test/e2e/templeton/build.xml
index 67968a3..f1361ea 100644
--- hcatalog/src/test/e2e/templeton/build.xml
+++ hcatalog/src/test/e2e/templeton/build.xml
@@ -107,6 +107,7 @@
+
diff --git hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
index 1e7cf36..54c365d 100644
--- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
+++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
@@ -681,7 +681,12 @@ sub compare
#try to get the call back url request until timeout
if ($result == 1 && defined $testCmd->{'check_call_back'}) {
my $d = $testCmd->{'http_daemon'};
- $d->timeout(300); #wait for 5 mins
+ if (defined $testCmd->{'timeout_seconds'}) {
+ $d->timeout($testCmd->{'timeout_seconds'})
+ }
+ else {
+ $d->timeout(300); #wait for 5 mins by default
+ }
my $url_requested;
$testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g;
print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n";
@@ -732,6 +737,10 @@ sub compare
my $jobComplete;
my $NUM_RETRIES = 60;
my $SLEEP_BETWEEN_RETRIES = 5;
+ if (defined $testCmd->{'timeout_seconds'} && $testCmd->{'timeout_seconds'} > 0) {
+ $SLEEP_BETWEEN_RETRIES = ($testCmd->{'timeout_seconds'} / $NUM_RETRIES);
+ print $log "found timeout_seconds & set SLEEP_BETWEEN_RETRIES=$SLEEP_BETWEEN_RETRIES";
+ }
#first wait for job completion
while ($NUM_RETRIES-- > 0) {
diff --git hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf
new file mode 100644
index 0000000..0e16b0c
--- /dev/null
+++ hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'TestHeartbeat',
+ 'tests' =>
+ [
+ {
+ # see HIVE-4808
+ # this is a long running test, takes 11 minutes
+ # -mt must be greater than mapred.task.timeout (600000ms)
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
+ 'post_options' => ['user.name=:UNAME:','arg=-mt', 'arg=642000',
+ 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=sleep', 'statusdir=/tmp' ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+#there seems to be a race condition: the test fails with exitValue: null even though if you check /tmp/exit
+# it contains the value.
+# 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ 'timeout_seconds' => 800,
+ },
+ ]
+ },
+##=============================================================================================================
+ ]
+},
+ ;
+
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java
index 7799d64..fc69529 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -44,7 +44,6 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.Mapper;
@@ -68,9 +67,6 @@
* in hdfs files.
*/
public class TempletonControllerJob extends Configured implements Tool {
- static enum ControllerCounters {SIMPLE_COUNTER}
-
- ;
public static final String COPY_NAME = "templeton.copy";
public static final String STATUSDIR_NAME = "templeton.statusdir";
public static final String JAR_ARGS_NAME = "templeton.args";
@@ -157,14 +153,13 @@ public void run(Context context)
conf.get(OVERRIDE_CLASSPATH));
String statusdir = conf.get(STATUSDIR_NAME);
- Counter cnt = context.getCounter(ControllerCounters.SIMPLE_COUNTER);
ExecutorService pool = Executors.newCachedThreadPool();
executeWatcher(pool, conf, context.getJobID(),
proc.getInputStream(), statusdir, STDOUT_FNAME);
executeWatcher(pool, conf, context.getJobID(),
proc.getErrorStream(), statusdir, STDERR_FNAME);
- KeepAlive keepAlive = startCounterKeepAlive(pool, cnt);
+ KeepAlive keepAlive = startCounterKeepAlive(pool, context);
proc.waitFor();
keepAlive.sendReport = false;
@@ -193,7 +188,7 @@ private void executeWatcher(ExecutorService pool, Configuration conf,
pool.execute(w);
}
- private KeepAlive startCounterKeepAlive(ExecutorService pool, Counter cnt)
+ private KeepAlive startCounterKeepAlive(ExecutorService pool, Context cnt)
throws IOException {
KeepAlive k = new KeepAlive(cnt);
pool.execute(k);
@@ -215,7 +210,7 @@ private void writeExitValue(Configuration conf, int exitValue, String statusdir)
}
}
- public static class Watcher implements Runnable {
+ private static class Watcher implements Runnable {
private InputStream in;
private OutputStream out;
private JobID jobid;
@@ -279,11 +274,11 @@ public void run() {
}
}
- public static class KeepAlive implements Runnable {
- private Counter cnt;
- public boolean sendReport;
+ private static class KeepAlive implements Runnable {
+ private final Mapper.Context cnt;
+ private volatile boolean sendReport;
- public KeepAlive(Counter cnt) {
+ public KeepAlive(Mapper.Context cnt) {
this.cnt = cnt;
this.sendReport = true;
}
@@ -292,7 +287,7 @@ public KeepAlive(Counter cnt) {
public void run() {
try {
while (sendReport) {
- cnt.increment(1);
+ cnt.progress();
Thread.sleep(KEEP_ALIVE_MSEC);
}
} catch (InterruptedException e) {