diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index df53406..cdef2e7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -754,6 +754,13 @@ public static final String YARN_APP_CONTAINER_LOG_SIZE = YARN_PREFIX + "app.container.log.filesize"; + // //////////////////////////////////// + // Application History Server configs + // /////////////////////////////////// + /** AHS STORAGE CLASS */ + public static final String AHS_STORAGE = + YARN_PREFIX + "yarn.ahs.storage.class"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistory.java new file mode 100644 index 0000000..2b0e2bc --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistory.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; + +public class ApplicationHistory extends AbstractService implements + ApplicationHistoryContext { + private static final Log LOG = LogFactory.getLog(ApplicationHistory.class); + + private ApplicationHistoryStore historyStore; + + public ApplicationHistory() { + super(ApplicationHistory.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + LOG.info("ApplicationHistory Init"); + historyStore = + ReflectionUtils.newInstance(conf.getClass( + YarnConfiguration.AHS_STORAGE, + FileSystemApplicationHistoryStore.class, + ApplicationHistoryStore.class), conf); + if (historyStore instanceof Service) { + ((Service) historyStore).init(conf); + } + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + LOG.info("Starting ApplicationHistory"); + if (historyStore instanceof Service) { + ((Service) historyStore).start(); + } + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + LOG.info("Stopping ApplicationHistory"); + if (historyStore != null && historyStore instanceof Service) { + ((Service) historyStore).stop(); + } + super.serviceStop(); + } + + @Override + public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) + throws IOException { + return getFinalContainerReport(historyStore.getAMContainer(appAttemptId)); + } + + @Override + public Map getAllApplications() + throws IOException { + Map histData = + historyStore.getAllApplications(); + HashMap applicationsHistory = + (histData instanceof HashMap) + ? (HashMap) histData + : new HashMap(histData); + HashMap applicationsReport = + new HashMap(); + for (ApplicationId appId : applicationsHistory.keySet()) { + applicationsReport.put(appId, + getFinalApplicationReport(applicationsHistory.get(appId))); + } + return applicationsReport; + } + + @Override + public ApplicationReport getApplication(ApplicationId appId) + throws IOException { + return getFinalApplicationReport(historyStore.getApplication(appId)); + } + + public ApplicationReport getFinalApplicationReport( + ApplicationHistoryData appHistory) { + return ApplicationReport.newInstance(appHistory.getApplicationId(), null, + appHistory.getUser(), appHistory.getQueue(), appHistory + .getApplicationName(), "", 0, null, null, "", "", appHistory + .getStartTime(), appHistory.getFinishTime(), appHistory + .getFinalApplicationStatus(), null, "", 100, appHistory + .getApplicationType(), null); + } + + public ApplicationAttemptReport getFinalApplicationAttemptReport( + ApplicationAttemptHistoryData appAttemptHistory) { + return ApplicationAttemptReport.newInstance(appAttemptHistory + .getApplicationAttemptId(), appAttemptHistory.getHost(), + appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(), + appAttemptHistory.getDiagnosticsInfo(), null, appAttemptHistory + .getMasterContainerId()); + } + + @Override + public ApplicationAttemptReport getApplicationAttempt( + ApplicationAttemptId appAttemptId) throws IOException { + return getFinalApplicationAttemptReport(historyStore + .getApplicationAttempt(appAttemptId)); + } + + @Override + public Map getApplicationAttempts( + ApplicationId appId) throws IOException { + Map histData = + historyStore.getApplicationAttempts(appId); + HashMap applicationAttemptsHistory = + (histData instanceof HashMap) + ? (HashMap) histData + : new HashMap( + histData); + HashMap applicationAttemptsReport = + new HashMap(); + for (ApplicationAttemptId appAttemptId : applicationAttemptsHistory + .keySet()) { + applicationAttemptsReport.put(appAttemptId, + getFinalApplicationAttemptReport(applicationAttemptsHistory + .get(appAttemptId))); + } + return applicationAttemptsReport; + } + + @Override + public ContainerReport getContainer(ContainerId containerId) + throws IOException { + return getFinalContainerReport(historyStore.getContainer(containerId)); + } + + public ContainerReport getFinalContainerReport( + ContainerHistoryData containerHistory) { + return ContainerReport + .newInstance(containerHistory.getContainerId(), containerHistory + .getAllocatedResource(), containerHistory.getAssignedNode(), + containerHistory.getPriority(), containerHistory.getStartTime(), + containerHistory.getFinishTime(), + containerHistory.getDiagnosticsInfo(), containerHistory.getLogURL(), + ContainerStatus.newInstance(containerHistory.getContainerId(), + containerHistory.getFinalContainerStatus(), containerHistory + .getDiagnosticsInfo(), 0)); + } + + @Override + public Map getContainers( + ApplicationAttemptId appAttemptId) throws IOException { + Map histData = + historyStore.getContainers(appAttemptId); + HashMap containersHistory = + (histData instanceof HashMap) + ? (HashMap) histData + : new HashMap(histData); + + HashMap containersReport = + new HashMap(); + for (ContainerId container : containersHistory.keySet()) { + containersReport.put(container, getFinalContainerReport(containersHistory + .get(container))); + } + return containersReport; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryContext.java new file mode 100644 index 0000000..4c0ac11 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryContext.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface ApplicationHistoryContext { + /** + * This method returns Application {@link ApplicationReport} for the specified + * {@link ApplicationId}. + * + * @return {@link ApplicationReport} for the ApplicationId. + * @throws {@link IOException} + */ + ApplicationReport getApplication(ApplicationId appId) throws IOException; + + /** + * This method returns all Application {@link ApplicationReport}s + * + * @return map {@link ApplicationId, @link ApplicationReport}s. + * @throws {@link IOException} + */ + Map getAllApplications() throws IOException; + + /** + * Application can have multiple application attempts + * {@link ApplicationAttemptReport}. This method returns the all + * {@link ApplicationAttemptReport}s for the Application. + * + * @return all {@link ApplicationAttemptReport}s for the Application. + * @throws {@link IOException} + */ + Map getApplicationAttempts( + ApplicationId appId) throws IOException; + + /** + * This method returns {@link ApplicationAttemptReport} for specified + * {@link ApplicationId}. + * + * @param {@link ApplicationAttemptId} + * @return {@link ApplicationAttemptReport} for ApplicationAttemptId + * @throws {@link IOException} + */ + ApplicationAttemptReport getApplicationAttempt( + ApplicationAttemptId appAttemptId) throws IOException; + + /** + * This method returns {@link ContainerReport} for specified {@link ContainerId}. + * + * @param {@link ContainerId} + * @return {@link Container} for ContainerId + * @throws {@link IOException} + */ + ContainerReport getContainer(ContainerId containerId) throws IOException; + + /** + * This method returns {@link ContainerReport} for specified + * {@link ApplicationAttemptId}. + * + * @param {@link ApplicationAttemptId} + * @return {@link Container} for ApplicationAttemptId + * @throws {@link IOException} + */ + ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) + throws IOException; + + /** + * This method returns Map{@link ContainerId,@link ContainerReport} for specified + * {@link ApplicationAttemptId}. + * + * @param {@link ApplicationAttemptId} + * @return Map{@link ContainerId, @link ContainerReport} for ApplicationAttemptId + * @throws {@link IOException} + */ + Map getContainers(ApplicationAttemptId appAttemptId) + throws IOException; +}