Index: hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
===================================================================
--- hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (revision 0)
+++ hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (revision 0)
@@ -0,0 +1,142 @@
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+
+##=============================================================================================================
+ {
+ 'name' => 'JOBS',
+ 'tests' =>
+ [
+ {
+ #a simple load store script as UNAME_OTHER
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #a simple load store script as UNAME
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #a simple load store script as UNAME_OTHER
+ 'num' => 3,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ # now we have 6 jobs (two of each previous test case), check if they have the right user tag
+ # GET jobs?user.name=UNAME_OTHER&showall=true&fields=*, should get all 6 jobs, with the right username field
+ 'num' => 4,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&showall=true&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME:',
+ '$[-4:].detail.status.username' => ':UNAME:', '$[-5:].detail.status.username' => ':UNAME_OTHER:', '$[-6:].detail.status.username' => ':UNAME_OTHER:'},
+ 'status_code' => 200,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER&fields=*, should get only jobs launched as UNAME_OTHER
+ 'num' => 5,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME_OTHER:',
+ '$[-4:].detail.status.username' => ':UNAME_OTHER:'},
+ 'status_code' => 200,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER, only get jobid but no detail
+ 'num' => 6,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].id' => 'job_.*', '$[-1:].detail' => '^$'},
+ 'status_code' => 200,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the job
+ 'num' => 7,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].id' => 'job_.*',
+ '$[-1:].detail.status.jobACLs' => '^.+$',
+ '$[-1:].detail.status.runState' => '\\d+',
+ '$[-1:].detail.status.jobId' => 'job_.*',
+ '$[-1:].detail.status.jobComplete' => 'true',
+ '$[-1:].detail.profile.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.profile.jobFile' => '^.+$',
+ '$[-1:].detail.profile.url' => '^.+$',
+ '$[-1:].detail.profile.queueName' => '^.+$',
+ '$[-1:].detail.profile.jobName' => 'loadstore\.pig',
+ '$[-1:].detail.profile.jobID.id' => '\\d+',
+ '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+',
+ '$[-1:].detail.profile.jobId' => 'job_.*',
+ '$[-1:].detail.id' => 'job_.*',
+ '$[-1:].detail.parentId' => 'job_.*',
+ '$[-1:].detail.percentComplete' => '100%',
+ '$[-1:].detail.exitValue' => '0',
+ '$[-1:].detail.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.callback' => '^.+$',
+ '$[-1:].detail.completed' => 'done',
+ },
+ 'status_code' => 200,
+ },
+ {
+ # GET queue?user.name=UNAME_OTHER, only get jobid as an array
+ 'num' => 8,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/queue?user.name=:UNAME_OTHER:',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:]' => 'job_.*'},
+ 'status_code' => 200,
+ },
+
+ ]
+ }
+ ]
+}
+;
Index: hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
===================================================================
--- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (revision 1477231)
+++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (working copy)
@@ -35,6 +35,7 @@
use English;
use Storable qw(dclone);
use File::Glob ':glob';
+use JSON::Path;
my $passedStr = 'passed';
my $failedStr = 'failed';
@@ -293,7 +294,16 @@
}
}
+ if (defined $testCmd->{$aPfix . 'json_path'}) {
+ my $json_path_matches = $testCmd->{$aPfix . 'json_path'};
+ my @keys = keys %{$json_path_matches};
+ foreach my $key (@keys) {
+ my $new_value = $self->replaceParametersInArg($json_path_matches->{$key}, $testCmd, $log);
+ $json_path_matches->{$key} = $new_value;
+ }
+ }
+
}
###############################################################################
@@ -590,6 +600,23 @@
my $json_hash;
my %json_info;
+ if (defined $testCmd->{'json_path'}) {
+ my $json_matches = $testCmd->{'json_path'};
+ foreach my $key (keys %$json_matches) {
+ my $regex_expected_value = $json_matches->{$key};
+ my $path = JSON::Path->new($key);
+ my $value = $path->value($testResult->{'body'});
+ if ($value !~ /$regex_expected_value/s) {
+ print $log "$0::$subName INFO check failed:"
+ . " json pattern check failed. For field "
+ . "$key, regex <" . $regex_expected_value
+ . "> did not match the result <" . $value
+ . ">\n";
+ $result = 0;
+ last;
+ }
+ }
+ }
if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) {
my $json = new JSON;
$json_hash = $json->utf8->decode($testResult->{'body'});
Index: hcatalog/src/test/e2e/templeton/build.xml
===================================================================
--- hcatalog/src/test/e2e/templeton/build.xml (revision 1477231)
+++ hcatalog/src/test/e2e/templeton/build.xml (working copy)
@@ -90,9 +90,11 @@
+
+
Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
===================================================================
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (revision 1477231)
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (working copy)
@@ -674,14 +674,57 @@
/**
* Return the status of the jobid.
+ * @deprecated use GET jobs/{jobid} instead.
*/
+ @Deprecated
@GET
@Path("queue/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
public QueueStatusBean showQueueId(@PathParam("jobid") String jobid)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
+ return showJobId(jobid);
+ }
+
+ /**
+ * Kill a job in the queue.
+ * @deprecated use DELETE jobs/{jobid} instead.
+ */
+ @Deprecated
+ @DELETE
+ @Path("queue/{jobid}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid)
+ throws NotAuthorizedException, BadParam, IOException, InterruptedException {
+ return deleteJobId(jobid);
+ }
+
+ /**
+ * Return all the known job ids for this user.
+ * @deprecated use GET jobs instead.
+ */
+ @Deprecated
+ @GET
+ @Path("queue")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List showQueueList()
+ throws NotAuthorizedException, BadParam, IOException, InterruptedException {
verifyUser();
+
+ ListDelegator d = new ListDelegator(appConf);
+ return d.run(getUser());
+ }
+
+ /**
+ * Return the status of the jobid.
+ */
+ @GET
+ @Path("jobs/{jobid}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public QueueStatusBean showJobId(@PathParam("jobid") String jobid)
+ throws NotAuthorizedException, BadParam, IOException, InterruptedException {
+
+ verifyUser();
verifyParam(jobid, ":jobid");
StatusDelegator d = new StatusDelegator(appConf);
@@ -692,9 +735,9 @@
* Kill a job in the queue.
*/
@DELETE
- @Path("queue/{jobid}")
+ @Path("jobs/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
- public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid)
+ public QueueStatusBean deleteJobId(@PathParam("jobid") String jobid)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
verifyUser();
@@ -708,15 +751,35 @@
* Return all the known job ids for this user.
*/
@GET
- @Path("queue")
+ @Path("jobs")
@Produces({MediaType.APPLICATION_JSON})
- public List showQueueList()
+ public List showJobList(@QueryParam("fields") String fields)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
- verifyUser();
+ verifyUser();
+
+ boolean showDetails = false;
+ if (fields!=null && !fields.equals("*")) {
+ throw new BadParam("fields value other than * is not supported");
+ }
+ if (fields!=null && fields.equals("*")) {
+ showDetails = true;
+ }
- ListDelegator d = new ListDelegator(appConf);
- return d.run(getUser());
+ ListDelegator ld = new ListDelegator(appConf);
+ List list = ld.run(getUser());
+ List detailList = new ArrayList();
+ for (String job : list) {
+ JobItemBean jobItem = new JobItemBean();
+ jobItem.id = job;
+ if (showDetails) {
+ StatusDelegator sd = new StatusDelegator(appConf);
+ QueueStatusBean statusBean = sd.run(getUser(), job);
+ jobItem.detail = statusBean;
+ }
+ detailList.add(jobItem);
+ }
+ return detailList;
}
/**
Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java
===================================================================
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java (revision 0)
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java (revision 0)
@@ -0,0 +1,20 @@
+package org.apache.hcatalog.templeton;
+
+public class JobItemBean {
+ public String id;
+ public QueueStatusBean detail;
+
+ public JobItemBean() {
+ }
+
+ /**
+ * Create a new JobItemBean
+ *
+ * @param id job id
+ * @param detail job detail
+ */
+ public JobItemBean(String id, QueueStatusBean detail) {
+ this.id = id;
+ this.detail = detail;
+ }
+}
Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java
===================================================================
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (revision 1477231)
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (working copy)
@@ -164,6 +164,8 @@
FilterMapping.REQUEST);
root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*",
FilterMapping.REQUEST);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*",
+ FilterMapping.REQUEST);
root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*",
FilterMapping.REQUEST);
root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*",