Index: hcatalog/src/test/e2e/templeton/tests/jobstatus.conf =================================================================== --- hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (revision 1536446) +++ hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (working copy) @@ -152,6 +152,36 @@ 'json_path' => {'$[-1:]' => 'job_.*'}, 'status_code' => 200, }, + { + # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the oldest 2 jobs whose + # id is greater than job_0 + 'num' => 9, + 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*&numrecords=2&jobid=job_0', + 'format_header' => 'Content-Type: application/json', + 'json_path' => {'$[-1:].id' => 'job_.*', + '$[-1:].detail.status.jobId' => 'job_.*', + '$[-1:].detail.status.runState' => '\\d+', + '$[-1:].detail.status.jobId' => 'job_.*', + '$[-1:].detail.status.jobComplete' => 'true', + '$[-1:].detail.profile.user' => ':UNAME_OTHER:', + '$[-1:].detail.profile.jobFile' => '^.+$', + '$[-1:].detail.profile.url' => '^.+$', + '$[-1:].detail.profile.queueName' => '^.+$', + '$[-1:].detail.profile.jobID.id' => '\\d+', + '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+', + '$[-1:].detail.profile.jobId' => 'job_.*', + '$[-1:].detail.id' => 'job_.*', + '$[-1:].detail.parentId' => 'job_.*', + '$[-1:].detail.percentComplete' => '100%', + '$[-1:].detail.exitValue' => '0', + '$[-1:].detail.user' => ':UNAME_OTHER:', + '$[-1:].detail.callback' => '^.+$', + '$[-1:].detail.completed' => 'done', + }, + 'status_code' => 200, + }, ] } Index: hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java =================================================================== --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (revision 1536446) +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (working copy) @@ -855,15 +855,59 @@ } /** - * Return all the known job ids for this user. + * Return all the known job ids for this user based on the optional filter conditions. + *

+ * Example usages: + * 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan' + * Return all the Job IDs submitted by hsubramaniyan + * 2. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&showall=true' + * Return all the Job IDs that are visible to hsubramaniyan + * 3. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&jobid=job_201312091733_0003' + * Return all the Job IDs for hsubramaniyan after job_201312091733_0003. + * 4. curl -s 'http://localhost:50111/templeton/v1/jobs? + * user.name=hsubramaniyan&jobid=job_201312091733_0003&numrecords=5' + * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003. + * 5. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&numrecords=5' + * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list + * lexicographically. + *

+ *

+ * Supporting pagination using "jobid" and "numrecords" parameters: + * Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n + * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid" + * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's + * last record and get the Job Id of the last record as job_yyy_k, else quit. + * Step 4: set "jobid"=job_yyy_k and go to step 2. + *

+ * @param fields If "fields" set to "*", the request will return full details of the job. + * If "fields" is missing, will only return the job ID. Currently the value can only + * be "*", other values are not allowed and will throw exception. + * @param showall If "showall" is set to "true", the request will return all jobs the user + * has permission to view, not only the jobs belonging to the user. + * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater + * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001", + * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of + * records returned depends on the value of "numrecords". + * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords + * records appearing after "jobid" will be returned after sorting the Job Id list + * lexicographically. + * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will + * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present + * and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are returned. + * @return list of job items based on the filter conditions specified by the user. */ @GET @Path("jobs") @Produces({MediaType.APPLICATION_JSON}) public List showJobList(@QueryParam("fields") String fields, - @QueryParam("showall") boolean showall) + @QueryParam("showall") boolean showall, + @QueryParam("jobid") String jobid, + @QueryParam("numrecords") String numrecords) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - + verifyUser(); boolean showDetails = false; @@ -877,7 +921,46 @@ ListDelegator ld = new ListDelegator(appConf); List list = ld.run(getDoAsUser(), showall); List detailList = new ArrayList(); + int currRecord = 0; + int numRecords; + + // Parse numrecords to an integer + try { + if (numrecords != null) { + numRecords = Integer.parseInt(numrecords); + if (numRecords <= 0) { + throw new BadParam("numrecords should be an integer > 0"); + } + } + else { + numRecords = -1; + } + } + catch(Exception e) { + throw new BadParam("Invalid numrecords format: numrecords should be an integer > 0"); + } + + // Sort the list lexicographically + Collections.sort(list); + for (String job : list) { + // If numRecords = -1, fetch all records. + // Hence skip all the below checks when numRecords = -1. + if (numRecords != -1) { + // If currRecord >= numRecords, we have already fetched the top #numRecords + if (currRecord >= numRecords) { + break; + } + // If the current record needs to be returned based on the + // filter conditions specified by the user, increment the counter + else if ((jobid != null && job.compareTo(jobid) > 0) || jobid == null) { + currRecord++; + } + // The current record should not be included in the output detailList. + else { + continue; + } + } JobItemBean jobItem = new JobItemBean(); jobItem.id = job; if (showDetails) {