Index: hcatalog/src/test/e2e/templeton/tests/jobstatus.conf =================================================================== --- hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (revision 0) +++ hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (revision 0) @@ -0,0 +1,142 @@ +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ + +##============================================================================================================= + { + 'name' => 'JOBS', + 'tests' => + [ + { + #a simple load store script as UNAME_OTHER + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #a simple load store script as UNAME + 'num' => 2, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + #a simple load store script as UNAME_OTHER + 'num' => 3, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + { + # now we have 6 jobs (two of each previous test case), check if they have the right user tag + # GET jobs?user.name=UNAME_OTHER&showall=true&fields=*, should get all 6 jobs, with the right username field + 'num' => 4, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&showall=true&fields=*', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME:', + '$[-4:].detail.status.username' => ':UNAME:', '$[-5:].detail.status.username' => ':UNAME_OTHER:', '$[-6:].detail.status.username' => ':UNAME_OTHER:'}, + 'status_code' => 200, + }, + { + # GET jobs?user.name=UNAME_OTHER&fields=*, should get only jobs launched as UNAME_OTHER + 'num' => 5, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME_OTHER:', + '$[-4:].detail.status.username' => ':UNAME_OTHER:'}, + 'status_code' => 200, + }, + { + # GET jobs?user.name=UNAME_OTHER, only get jobid but no detail + 'num' => 6, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:].id' => 'job_.*', '$[-1:].detail' => '^$'}, + 'status_code' => 200, + }, + { + # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the job + 'num' => 7, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:].id' => 'job_.*', + '$[-1:].detail.status.jobACLs' => '^.+$', + '$[-1:].detail.status.runState' => '\\d+', + '$[-1:].detail.status.jobId' => 'job_.*', + '$[-1:].detail.status.jobComplete' => 'true', + '$[-1:].detail.profile.user' => ':UNAME_OTHER:', + '$[-1:].detail.profile.jobFile' => '^.+$', + '$[-1:].detail.profile.url' => '^.+$', + '$[-1:].detail.profile.queueName' => '^.+$', + '$[-1:].detail.profile.jobName' => 'loadstore\.pig', + '$[-1:].detail.profile.jobID.id' => '\\d+', + '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+', + '$[-1:].detail.profile.jobId' => 'job_.*', + '$[-1:].detail.id' => 'job_.*', + '$[-1:].detail.parentId' => 'job_.*', + '$[-1:].detail.percentComplete' => '100%', + '$[-1:].detail.exitValue' => '0', + '$[-1:].detail.user' => ':UNAME_OTHER:', + '$[-1:].detail.callback' => '^.+$', + '$[-1:].detail.completed' => 'done', + }, + 'status_code' => 200, + }, + { + # GET queue?user.name=UNAME_OTHER, only get jobid as an array + 'num' => 8, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/queue?user.name=:UNAME_OTHER:', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:]' => 'job_.*'}, + 'status_code' => 200, + }, + + ] + } + ] +} +; Index: hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm =================================================================== --- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (revision 1477231) +++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (working copy) @@ -35,6 +35,7 @@ use English; use Storable qw(dclone); use File::Glob ':glob'; +use JSON::Path; my $passedStr = 'passed'; my $failedStr = 'failed'; @@ -293,7 +294,16 @@ } } + if (defined $testCmd->{$aPfix . 'json_path'}) { + my $json_path_matches = $testCmd->{$aPfix . 'json_path'}; + my @keys = keys %{$json_path_matches}; + foreach my $key (@keys) { + my $new_value = $self->replaceParametersInArg($json_path_matches->{$key}, $testCmd, $log); + $json_path_matches->{$key} = $new_value; + } + } + } ############################################################################### @@ -590,6 +600,23 @@ my $json_hash; my %json_info; + if (defined $testCmd->{'json_path'}) { + my $json_matches = $testCmd->{'json_path'}; + foreach my $key (keys %$json_matches) { + my $regex_expected_value = $json_matches->{$key}; + my $path = JSON::Path->new($key); + my $value = $path->value($testResult->{'body'}); + if ($value !~ /$regex_expected_value/s) { + print $log "$0::$subName INFO check failed:" + . " json pattern check failed. For field " + . "$key, regex <" . $regex_expected_value + . "> did not match the result <" . $value + . ">\n"; + $result = 0; + last; + } + } + } if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) { my $json = new JSON; $json_hash = $json->utf8->decode($testResult->{'body'}); Index: hcatalog/src/test/e2e/templeton/build.xml =================================================================== --- hcatalog/src/test/e2e/templeton/build.xml (revision 1477231) +++ hcatalog/src/test/e2e/templeton/build.xml (working copy) @@ -90,9 +90,11 @@ + + Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (revision 1477231) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (working copy) @@ -674,14 +674,57 @@ /** * Return the status of the jobid. + * @deprecated use GET jobs/{jobid} instead. */ + @Deprecated @GET @Path("queue/{jobid}") @Produces({MediaType.APPLICATION_JSON}) public QueueStatusBean showQueueId(@PathParam("jobid") String jobid) throws NotAuthorizedException, BadParam, IOException, InterruptedException { + return showJobId(jobid); + } + + /** + * Kill a job in the queue. + * @deprecated use DELETE jobs/{jobid} instead. + */ + @Deprecated + @DELETE + @Path("queue/{jobid}") + @Produces({MediaType.APPLICATION_JSON}) + public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + return deleteJobId(jobid); + } + + /** + * Return all the known job ids for this user. + * @deprecated use GET jobs instead. + */ + @Deprecated + @GET + @Path("queue") + @Produces({MediaType.APPLICATION_JSON}) + public List showQueueList() + throws NotAuthorizedException, BadParam, IOException, InterruptedException { verifyUser(); + + ListDelegator d = new ListDelegator(appConf); + return d.run(getUser()); + } + + /** + * Return the status of the jobid. + */ + @GET + @Path("jobs/{jobid}") + @Produces({MediaType.APPLICATION_JSON}) + public QueueStatusBean showJobId(@PathParam("jobid") String jobid) + throws NotAuthorizedException, BadParam, IOException, InterruptedException { + + verifyUser(); verifyParam(jobid, ":jobid"); StatusDelegator d = new StatusDelegator(appConf); @@ -692,9 +735,9 @@ * Kill a job in the queue. */ @DELETE - @Path("queue/{jobid}") + @Path("jobs/{jobid}") @Produces({MediaType.APPLICATION_JSON}) - public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) + public QueueStatusBean deleteJobId(@PathParam("jobid") String jobid) throws NotAuthorizedException, BadParam, IOException, InterruptedException { verifyUser(); @@ -708,15 +751,35 @@ * Return all the known job ids for this user. */ @GET - @Path("queue") + @Path("jobs") @Produces({MediaType.APPLICATION_JSON}) - public List showQueueList() + public List showJobList(@QueryParam("fields") String fields) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - verifyUser(); + verifyUser(); + + boolean showDetails = false; + if (fields!=null && !fields.equals("*")) { + throw new BadParam("fields value other than * is not supported"); + } + if (fields!=null && fields.equals("*")) { + showDetails = true; + } - ListDelegator d = new ListDelegator(appConf); - return d.run(getUser()); + ListDelegator ld = new ListDelegator(appConf); + List list = ld.run(getUser()); + List detailList = new ArrayList(); + for (String job : list) { + JobItemBean jobItem = new JobItemBean(); + jobItem.id = job; + if (showDetails) { + StatusDelegator sd = new StatusDelegator(appConf); + QueueStatusBean statusBean = sd.run(getUser(), job); + jobItem.detail = statusBean; + } + detailList.add(jobItem); + } + return detailList; } /** Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java (revision 0) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JobItemBean.java (revision 0) @@ -0,0 +1,20 @@ +package org.apache.hcatalog.templeton; + +public class JobItemBean { + public String id; + public QueueStatusBean detail; + + public JobItemBean() { + } + + /** + * Create a new JobItemBean + * + * @param id job id + * @param detail job detail + */ + public JobItemBean(String id, QueueStatusBean detail) { + this.id = id; + this.detail = detail; + } +} Index: hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (revision 1477231) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (working copy) @@ -164,6 +164,8 @@ FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", + FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", FilterMapping.REQUEST); root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*",