diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationsQuery.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationsQuery.java new file mode 100644 index 0000000..2791393 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationsQuery.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import java.util.EnumSet; +import java.util.Set; + +import org.apache.commons.lang.math.LongRange; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable +public class ApplicationsQuery { + + private Set applicationTypes; + + private EnumSet applicationStates; + + private Set users; + + private Set queues; + + private long limit; + + private LongRange start; + + private LongRange finish; + + @Public + @Unstable + public static ApplicationsQuery newInstance( + Set applicationTypes, EnumSet applicationStates, + Set users, Set queues, long limit, LongRange start, + LongRange finish) { + ApplicationsQuery query = new ApplicationsQuery(); + query.setApplicationTypes(applicationTypes); + query.setApplicationStates(applicationStates); + query.setUsers(users); + query.setQueues(queues); + query.setLimit(limit); + query.setStart(start); + query.setFinish(finish); + return query; + } + + @Public + @Unstable + public Set getApplictaionTypes() { + return applicationTypes; + } + + @Public + @Unstable + public EnumSet getApplicationStates() { + return applicationStates; + } + + @Public + @Unstable + public Set getUsers() { + return users; + } + + @Public + @Unstable + public Set getQueues() { + return queues; + } + + @Public + @Unstable + public long getLimit() { + return limit; + } + + @Public + @Unstable + public LongRange getStart() { + return start; + } + + @Public + @Unstable + public LongRange getFinish() { + return finish; + } + + @Public + @Unstable + public void setApplicationTypes(Set applicationTypes) { + this.applicationTypes = applicationTypes; + } + + @Public + @Unstable + public void setApplicationStates( + EnumSet applicationStates) { + this.applicationStates = applicationStates; + } + + @Public + @Unstable + public void setUsers(Set users) { + this.users = users; + } + + @Public + @Unstable + public void setQueues(Set queues) { + this.queues = queues; + } + + @Public + @Unstable + public void setLimit(long limit) { + this.limit = limit; + } + + @Public + @Unstable + public void setStart(LongRange start) { + this.start = start; + } + + @Public + @Unstable + public void setFinish(LongRange finish) { + this.finish = finish; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 56558da..b450013 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -21,12 +21,17 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.lang.math.LongRange; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; @@ -50,12 +55,15 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; public class ApplicationHistoryClientService extends AbstractService { @@ -166,12 +174,36 @@ public GetApplicationReportResponse getApplicationReport( @Override public GetApplicationsResponse getApplications( GetApplicationsRequest request) throws YarnException, IOException { - GetApplicationsResponse response = - GetApplicationsResponse.newInstance(new ArrayList( - history.getAllApplications().values())); - return response; + return getApplications(request, true); } + @Private + public GetApplicationsResponse getApplications( + GetApplicationsRequest request, boolean caseSensitive) + throws YarnException, IOException { + + Set applicationTypes = request.getApplicationTypes(); + EnumSet applicationStates = + request.getApplicationStates(); + Set users = request.getUsers(); + Set queues = request.getQueues(); + long limit = request.getLimit(); + LongRange start = request.getStartRange(); + LongRange finish = request.getFinishRange(); + + ApplicationsQuery query = ApplicationsQuery.newInstance( + applicationTypes, applicationStates, users, queues, limit, start, finish); + List reports = + new ArrayList(history.getFilterApplications(query).values()); + GetApplicationsResponse response = + GetApplicationsResponse.newInstance(reports); + return response; + } + @Override public GetContainerReportResponse getContainerReport( GetContainerReportRequest request) throws YarnException, IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 1e13a23..2a8e4b2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -105,6 +106,22 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) } @Override + public Map getFilterApplications( + ApplicationsQuery query) throws IOException { + Map histData = + historyStore.getFilterApplications(query); + HashMap applicationsReport = + new HashMap(); + + for (Entry entry : histData + .entrySet()) { + applicationsReport.put(entry.getKey(), + convertToApplicationReport(entry.getValue())); + } + return applicationsReport; + } + + @Override public ApplicationReport getApplication(ApplicationId appId) throws IOException { return convertToApplicationReport(historyStore.getApplication(appId)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java index 590853a..392301b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; @@ -55,6 +56,16 @@ throws IOException; /** + * + * @param query + * + * @return map of {@link ApplicationId} to {@link ApplicationHistoryData}s. + * @throws IOException + */ + Map getFilterApplications( + ApplicationsQuery query) throws IOException; + + /** * Application can have multiple application attempts * {@link ApplicationAttemptHistoryData}. This method returns the all * {@link ApplicationAttemptHistoryData}s for the Application. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index 9109dfc..24007cd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -45,6 +45,7 @@ import org.apache.hadoop.io.file.tfile.TFile; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -211,6 +212,61 @@ public ApplicationHistoryData getApplication(ApplicationId appId) } @Override + public Map getFilterApplications( + ApplicationsQuery query) throws IOException { + + Map historyDataMap = + new HashMap(); + FileStatus[] files = fs.listStatus(rootDirPath); + for (FileStatus file : files) { + ApplicationId appId = + ConverterUtils.toApplicationId(file.getPath().getName()); + try { + ApplicationHistoryData historyData = getApplication(appId); + // Application Types check + if (query.getApplictaionTypes() != null && query.getApplictaionTypes().size() != 0 && + !query.getApplictaionTypes().contains(historyData.getApplicationType())) { + continue; + } + // Application States check + if (query.getApplicationStates() != null && query.getApplicationStates().size() != 0 && + !query.getApplicationStates().contains(historyData.getYarnApplicationState())) { + continue; + } + // Application User check + if (query.getUsers() != null && query.getUsers().size() != 0 && + !query.getUsers().contains(historyData.getUser())) { + continue; + } + // Application Queue check + if (query.getQueues() != null && query.getQueues().size() != 0 && + !query.getQueues().contains(historyData.getQueue())) { + continue; + } + // Application StartTime check + if (query.getStart() != null && !query.getStart().containsLong( + historyData.getStartTime())) { + continue; + } + // Application FinishTime check + if (query.getFinish() != null && !query.getFinish().containsLong( + historyData.getFinishTime())) { + continue; + } + historyDataMap.put(appId, historyData); + if (historyDataMap.size() >= query.getLimit()) { + break; + } + } catch (IOException e) { + LOG.error("History Information of application " + appId + + " is not included into the result due to the exception", e); + } + } + LOG.info("get " + historyDataMap.size() + " applications"); + return historyDataMap; + } + + @Override public Map getApplicationAttempts(ApplicationId appId) throws IOException { Map historyDataMap = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java index 916335e..8facd36 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java @@ -30,6 +30,7 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; @@ -70,6 +71,44 @@ public MemoryApplicationHistoryStore() { } @Override + public Map getFilterApplications( + ApplicationsQuery query) throws IOException { + Map historyData = + new HashMap(); + for (Map.Entry history : applicationData.entrySet()) { + if (query.getApplictaionTypes() != null && query.getApplictaionTypes().size() != 0 && + !query.getApplictaionTypes().contains(history.getValue().getApplicationType())) { + continue; + } + if (query.getApplicationStates() != null && query.getApplicationStates().size() != 0 && + !query.getApplicationStates().contains(history.getValue().getYarnApplicationState())) { + continue; + } + if (query.getUsers() != null && query.getUsers().size() != 0 && + !query.getUsers().contains(history.getValue().getUser())) { + continue; + } + if (query.getQueues() != null && query.getQueues().size() != 0 && + !query.getQueues().contains(history.getValue().getQueue())) { + continue; + } + if (query.getStart() != null && !query.getStart().containsLong( + history.getValue().getStartTime())) { + continue; + } + if (query.getFinish() != null && !query.getFinish().containsLong( + history.getValue().getFinishTime())) { + continue; + } + historyData.put(history.getKey(), history.getValue()); + if (historyData.size() >= query.getLimit()) { + break; + } + } + return historyData; + } + + @Override public ApplicationHistoryData getApplication(ApplicationId appId) { return applicationData.get(appId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/NullApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/NullApplicationHistoryStore.java index 3660c10..8e23f8d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/NullApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/NullApplicationHistoryStore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; @@ -95,6 +96,12 @@ public ApplicationHistoryData getApplication(ApplicationId appId) } @Override + public Map getFilterApplications( + ApplicationsQuery query) throws IOException { + return Collections.emptyMap(); + } + + @Override public Map getApplicationAttempts(ApplicationId appId) throws IOException { return Collections.emptyMap(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index c31efab..6074c2b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -20,15 +20,19 @@ import java.io.IOException; import java.net.URI; +import java.util.HashSet; +import java.util.Set; import junit.framework.Assert; +import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -69,6 +73,7 @@ public void tearDown() throws Exception { public void testReadWriteHistoryData() throws IOException { testWriteHistoryData(5); testReadHistoryData(5); + testFilterReadHistoryData(2); } private void testWriteHistoryData(int num) throws IOException { @@ -193,4 +198,15 @@ public void testMassiveWriteContainerHistoryData() throws IOException { Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); } + private void testFilterReadHistoryData(int num) throws IOException { + Set users = new HashSet(); + users.add("test user"); + Set queues = new HashSet(); + queues.add("test queue"); + LongRange start = new LongRange(0, Long.MAX_VALUE); + LongRange finish = new LongRange(0, Long.MAX_VALUE); + ApplicationsQuery query = ApplicationsQuery.newInstance(null, null, users, queues, 2, start, finish); + Assert.assertEquals(num, store.getFilterApplications(query).size()); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java index 78ae0dd..15bed36 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationsQuery; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -56,6 +57,18 @@ @Unstable Map getAllApplications() throws IOException; + + /** + * This method returns filtered Application {@link ApplicationReport}s + * + * @return map of {@link ApplicationId} to {@link ApplicationReport}s. + * @throws IOException + */ + @Public + @Unstable + Map getFilterApplications( + ApplicationsQuery query) throws IOException; + /** * Application can have multiple application attempts * {@link ApplicationAttemptReport}. This method returns the all