diff --git hcatalog/src/test/e2e/templeton/README.txt hcatalog/src/test/e2e/templeton/README.txt index dac6ffc..647cfc1 100644 --- hcatalog/src/test/e2e/templeton/README.txt +++ hcatalog/src/test/e2e/templeton/README.txt @@ -150,3 +150,14 @@ Also useful to add to conf/hadoop-env.sh export HADOOP_OPTS="-Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk" to prevent warning about SCDynamicStore which may throw some tests off (http://stackoverflow.com/questions/7134723/hadoop-on-osx-unable-to-load-realm-info-from-scdynamicstore) + + +Performance +----------- +It's a good idea to set fork.factor.conf.file={number of .conf files} and fork.factor.group to something > 1 +(see build.xml) to make these tests run faster. If doing this, make sure the Hadoop Cluster has +enough map slots (10?) (mapred.tasktracker.map.tasks.maximum), otherwise test parallelism won't help. + +Adding Tests +------------ +ToDo: add some guidelines diff --git hcatalog/src/test/e2e/templeton/build.xml hcatalog/src/test/e2e/templeton/build.xml index 67968a3..f1361ea 100644 --- hcatalog/src/test/e2e/templeton/build.xml +++ hcatalog/src/test/e2e/templeton/build.xml @@ -107,6 +107,7 @@ + diff --git hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm index 1e7cf36..54c365d 100644 --- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm +++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm @@ -681,7 +681,12 @@ sub compare #try to get the call back url request until timeout if ($result == 1 && defined $testCmd->{'check_call_back'}) { my $d = $testCmd->{'http_daemon'}; - $d->timeout(300); #wait for 5 mins + if (defined $testCmd->{'timeout_seconds'}) { + $d->timeout($testCmd->{'timeout_seconds'}) + } + else { + $d->timeout(300); #wait for 5 mins by default + } my $url_requested; $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g; print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n"; @@ -732,6 +737,10 @@ sub compare my $jobComplete; my $NUM_RETRIES = 60; my $SLEEP_BETWEEN_RETRIES = 5; + if (defined $testCmd->{'timeout_seconds'} && $testCmd->{'timeout_seconds'} > 0) { + $SLEEP_BETWEEN_RETRIES = ($testCmd->{'timeout_seconds'} / $NUM_RETRIES); + print $log "found timeout_seconds & set SLEEP_BETWEEN_RETRIES=$SLEEP_BETWEEN_RETRIES"; + } #first wait for job completion while ($NUM_RETRIES-- > 0) { diff --git hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf new file mode 100644 index 0000000..0e16b0c --- /dev/null +++ hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'TestHeartbeat', + 'tests' => + [ + { + # see HIVE-4808 + # this is a long running test, takes 11 minutes + # -mt must be greater than mapred.task.timeout (600000ms) + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar', + 'post_options' => ['user.name=:UNAME:','arg=-mt', 'arg=642000', + 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=sleep', 'statusdir=/tmp' ], + 'json_field_substr_match' => { 'id' => '\d+'}, + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', +#there seems to be a race condition: the test fails with exitValue: null even though if you check /tmp/exit +# it contains the value. +# 'check_job_exit_value' => 0, + 'check_call_back' => 1, + 'timeout_seconds' => 800, + }, + ] + }, +##============================================================================================================= + ] +}, + ; + diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java index 7799d64..fc69529 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java @@ -44,7 +44,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Mapper; @@ -68,9 +67,6 @@ * in hdfs files. */ public class TempletonControllerJob extends Configured implements Tool { - static enum ControllerCounters {SIMPLE_COUNTER} - - ; public static final String COPY_NAME = "templeton.copy"; public static final String STATUSDIR_NAME = "templeton.statusdir"; public static final String JAR_ARGS_NAME = "templeton.args"; @@ -157,14 +153,13 @@ public void run(Context context) conf.get(OVERRIDE_CLASSPATH)); String statusdir = conf.get(STATUSDIR_NAME); - Counter cnt = context.getCounter(ControllerCounters.SIMPLE_COUNTER); ExecutorService pool = Executors.newCachedThreadPool(); executeWatcher(pool, conf, context.getJobID(), proc.getInputStream(), statusdir, STDOUT_FNAME); executeWatcher(pool, conf, context.getJobID(), proc.getErrorStream(), statusdir, STDERR_FNAME); - KeepAlive keepAlive = startCounterKeepAlive(pool, cnt); + KeepAlive keepAlive = startCounterKeepAlive(pool, context); proc.waitFor(); keepAlive.sendReport = false; @@ -193,7 +188,7 @@ private void executeWatcher(ExecutorService pool, Configuration conf, pool.execute(w); } - private KeepAlive startCounterKeepAlive(ExecutorService pool, Counter cnt) + private KeepAlive startCounterKeepAlive(ExecutorService pool, Context cnt) throws IOException { KeepAlive k = new KeepAlive(cnt); pool.execute(k); @@ -215,7 +210,7 @@ private void writeExitValue(Configuration conf, int exitValue, String statusdir) } } - public static class Watcher implements Runnable { + private static class Watcher implements Runnable { private InputStream in; private OutputStream out; private JobID jobid; @@ -279,11 +274,11 @@ public void run() { } } - public static class KeepAlive implements Runnable { - private Counter cnt; - public boolean sendReport; + private static class KeepAlive implements Runnable { + private final Mapper.Context cnt; + private volatile boolean sendReport; - public KeepAlive(Counter cnt) { + public KeepAlive(Mapper.Context cnt) { this.cnt = cnt; this.sendReport = true; } @@ -292,7 +287,7 @@ public KeepAlive(Counter cnt) { public void run() { try { while (sendReport) { - cnt.increment(1); + cnt.progress(); Thread.sleep(KEEP_ALIVE_MSEC); } } catch (InterruptedException e) {